import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sns
import importlib

import Notebooks.utils.utils as utils
import Notebooks.utils.da_utils as da_utils
import src.ds_utils.dataset as dataset

import src.models.boost_ensemble.ensemble_wrapper as ensemble_wrapper
import src.models.shared.shap_analysis as shap

sns.set_theme(style="darkgrid", palette="pastel")
plt.style.use("fivethirtyeight")

VERBOSE = True

utils.fix_cwd()

importlib.reload(dataset)
ds = dataset.create_dataset(cache_path="dataset_cache/anonymize_cached_44898_default_80.csv")

if VERBOSE:
    display(ds.full_data.sample(200))

if VERBOSE:
    print(len(ds.full_data.dtypes))
    with pd.option_context('display.max_rows', None):
        dtype_df = pd.DataFrame({'Column': ds.full_data.dtypes.index, 'Type': ds.full_data.dtypes.values})
        display(dtype_df)

    # display(ds.full_data.dtypes)

289

# print(len(ds.full_data))

importlib.reload(da_utils)
da_utils.plot_text_size_distributions(ds.full_data, text_col="text", token_counts=ds.full_data["num_tokens"])

importlib.reload(da_utils)
da_utils.plot_stacked_kde_text_length_vs_target(df=ds.full_data, len_column="num_tokens", target_column="fake")

<Axes: title={'center': 'Text Size (Spacy Token Count)'}, xlabel='Text Length', ylabel='Density'>

# 3. Proportion of "fake" for binary columns (prefixed with person_, org_, gpe_, etc.) including count
binary_columns = ds.full_data.columns[ds.full_data.columns.str.startswith(('person_', 'org_', 'gpe_'))]
fake_proportion_binary = ds.full_data[binary_columns].mean().reset_index()

fake_proportion_binary.columns = ['binary_column', 'total_proportion']
fake_proportion_binary['total_count'] = ds.full_data[binary_columns].sum().values

d = []
for col in binary_columns:
    d.append({
        'col': col,
        'total': len(ds.full_data[ds.full_data[col] == True]),
        'fake': len(ds.full_data[(ds.full_data[col] == True) & (ds.full_data["fake"] == True)]),
        'true': len(ds.full_data[(ds.full_data[col] == True) & (ds.full_data["fake"] == False)])
    })
d = pd.DataFrame(d)
d["prop_fake"] = d["fake"] / d["total"]

d.sort_values(by="total", ascending=False)

total_samples = len(ds.full_data)
overall_fake_proportion = ds.full_data['fake'].mean()

d['total_normalized'] = d['total'] / total_samples
d['prop_fake_normalized'] = d['prop_fake'] - overall_fake_proportion

# Create importance score (you can adjust the weights if needed)
d['importance_score'] = 0.5 * d['total_normalized'] + 0.5 * abs(d['prop_fake_normalized'])

# Sort by importance score
d_sorted = d.sort_values('importance_score', ascending=False)

plt.figure(figsize=(12, 8))
sns.barplot(x='importance_score', y='col', data=d_sorted.head(20))
plt.title('Top 20 Entities by Importance Score')
plt.xlabel('Importance Score')
plt.ylabel('Entity')
plt.tight_layout()
plt.show()

plt.figure(figsize=(12, 8))
sns.scatterplot(x='total', y='prop_fake', data=d, hue='importance_score', size='importance_score', sizes=(20, 200))
plt.title('Total Count vs. Proportion of Fake News')
plt.xlabel('Total Count')
plt.ylabel('Proportion of Fake News')
for i, row in d_sorted.head(10).iterrows():
    plt.annotate(row['col'], (row['total'], row['prop_fake']))
plt.tight_layout()
plt.show()

top_30 = d_sorted.head(30).set_index('col')
plt.figure(figsize=(12, 10))
sns.heatmap(top_30[['total_normalized', 'prop_fake_normalized']], annot=True, cmap='YlOrRd')
plt.title('Top 30 Entities: Normalized Total Count and Fake News Proportion')
plt.tight_layout()
plt.show()

# entity_type_summary = d.groupby('entity_type').agg({
#     'total': 'sum',
#     'prop_fake': 'mean',
#     'importance_score': 'mean'
# }).sort_values('importance_score', ascending=False)
# 
# entity_type_summary

correlation_matrix = d[['total', 'prop_fake', 'importance_score']].corr()
correlation_matrix

numeric_feats = ds.full_data.select_dtypes(include=[np.number])
# float_columns = df.select_dtypes(include=['float'])

numeric_feats

from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA

# Drop the 'fake' column if it exists
if 'fake' in numeric_feats.columns:
    df = numeric_feats.drop('fake', axis=1)

# Prepare the data for PCA
X = numeric_feats.select_dtypes(include=[np.number])
X_scaled = StandardScaler().fit_transform(X)

# Perform PCA
pca = PCA()
pca.fit(X_scaled)

# Calculate cumulative explained variance ratio
cumulative_variance_ratio = np.cumsum(pca.explained_variance_ratio_)

# Plot cumulative explained variance
plt.figure(figsize=(10, 6))
plt.plot(range(1, len(cumulative_variance_ratio) + 1), cumulative_variance_ratio, 'bo-')
plt.xlabel('Number of Components')
plt.ylabel('Cumulative Explained Variance Ratio')
plt.title('Cumulative Explained Variance vs. Number of Components')
plt.grid(True)
plt.tight_layout()
plt.show()

# Print the number of components needed to explain 95% of the variance
n_components_95 = np.argmax(cumulative_variance_ratio >= 0.95) + 1
print(f"Number of components needed to explain 95% of the variance: {n_components_95}")

Number of components needed to explain 95% of the variance: 25

from sklearn.manifold import TSNE

# T-SNE
input_df = numeric_feats.copy()

# Drop the 'fake' column if it exists
if 'fake' in input_df.columns:
    input_df = input_df.drop('fake', axis=1)

# Prepare the data for t-SNE
features_for_tsne = input_df.select_dtypes(include=[np.number])  # Select only numeric columns
scaled_features = StandardScaler().fit_transform(features_for_tsne)

# Perform t-SNE
tsne = TSNE(n_components=2, random_state=42)
tsne_results = tsne.fit_transform(scaled_features)

import pandas as pd
import numpy as np
from sklearn.preprocessing import StandardScaler
from sklearn.cluster import KMeans
from sklearn.metrics import silhouette_score
import matplotlib.pyplot as plt
import time

data = numeric_feats.copy()
# Remove the 'fake' column
features = data.drop('fake', axis=1)

# Standardize the features
scaler = StandardScaler()
scaled_features = scaler.fit_transform(features)

# Function to perform K-means clustering
def perform_kmeans(data, max_clusters=10):
    inertias = []
    silhouette_scores = []
    times = []

    for k in range(2, max_clusters + 1):
        start_time = time.time()
        kmeans = KMeans(n_clusters=k, n_init=10, random_state=42)
        kmeans.fit(data)
        end_time = time.time()

        inertias.append(kmeans.inertia_)
        silhouette_scores.append(silhouette_score(data, kmeans.labels_))
        times.append(end_time - start_time)

    return inertias, silhouette_scores, times

# Perform K-means for different numbers of clusters
inertias, silhouette_scores, times = perform_kmeans(scaled_features)

# Plot the elbow curve
plt.figure(figsize=(12, 4))
plt.subplot(131)
plt.plot(range(2, len(inertias) + 2), inertias, marker='o')
plt.xlabel('Number of clusters')
plt.ylabel('Inertia')
plt.title('Elbow Method')

# Plot the silhouette scores
plt.subplot(132)
plt.plot(range(2, len(silhouette_scores) + 2), silhouette_scores, marker='o')
plt.xlabel('Number of clusters')
plt.ylabel('Silhouette Score')
plt.title('Silhouette Analysis')

# Plot the execution times
plt.subplot(133)
plt.plot(range(2, len(times) + 2), times, marker='o')
plt.xlabel('Number of clusters')
plt.ylabel('Execution Time (seconds)')
plt.title('Performance Analysis')

plt.tight_layout()
plt.show()

# Determine the optimal number of clusters (you may need to adjust this based on the plots)
optimal_clusters = 10  # This is an example, adjust based on your analysis

# Perform final clustering with the optimal number of clusters
final_kmeans = KMeans(n_clusters=optimal_clusters, n_init=10, random_state=42)
cluster_labels = final_kmeans.fit_predict(scaled_features)

# Add cluster labels to the original dataset
data['Cluster'] = cluster_labels

# # Print summary statistics for each cluster
# for cluster in range(optimal_clusters):
#     print(f"\nCluster {cluster} Statistics:")
#     print(data[data['Cluster'] == cluster].describe())

# Visualize the clusters using the first two principal components (optional)
from sklearn.decomposition import PCA

pca = PCA(n_components=2)
pca_result = pca.fit_transform(scaled_features)

plt.figure(figsize=(10, 8))
scatter = plt.scatter(pca_result[:, 0], pca_result[:, 1], c=cluster_labels, cmap='viridis')
plt.colorbar(scatter)
plt.xlabel('First Principal Component')
plt.ylabel('Second Principal Component')
plt.title('Cluster Visualization using PCA')
plt.show()

# Create a scatter plot of the t-SNE results
plt.figure(figsize=(10, 8))
scatter = plt.scatter(tsne_results[:, 0],
                      tsne_results[:, 1], 
                      alpha=0.75,
                      s=1,
                      c=cluster_labels, 
                      # c=features_for_tsne['sentiment'], 
                      
                      cmap='viridis')
plt.colorbar(scatter, label='Fake News')
plt.title('t-SNE Visualization of Clusters')
plt.xlabel('t-SNE feature 1')
plt.ylabel('t-SNE feature 2')
plt.tight_layout()
plt.show()

# Print some statistics about the t-SNE results
print("t-SNE Results Shape:", tsne_results.shape)
print("t-SNE Range - X:", tsne_results[:, 0].min(), "to", tsne_results[:, 0].max())
print("t-SNE Range - Y:", tsne_results[:, 1].min(), "to", tsne_results[:, 1].max())

# Optionally, you can save the t-SNE results back to the dataframe
input_df['tsne_1'] = tsne_results[:, 0]
input_df['tsne_2'] = tsne_results[:, 1]

t-SNE Results Shape: (44898, 2)
t-SNE Range - X: -94.89072 to 95.122025
t-SNE Range - Y: -95.44623 to 91.66013

USE FASTTEXT EMBEDDINGS (unsupervised) for clustering
    SEE:
    
    https://chatgpt.com/c/8bbeec1a-00e9-4840-b874-8cb225ca2fbc
    
    !!

OR BETTER USE BERT EMBEDDINGS

BERT EMBEDDINGS:

T-SNE and CLUSTER

/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py:1601: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be depracted in transformers v4.45, and will be then set to `False` by default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884
  warnings.warn(
A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.
A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.
A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.
A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.
A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.
A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.
A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.
A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.
A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.
A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.
A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.
A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.
A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.
A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.
A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.
A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.
A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.
A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.
A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.
A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.
A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.
A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.
A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.
A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.
A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.
A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.
A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.
A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.
A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.
A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.
A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.
A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.
A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.
A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.
A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.
A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.
A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.
A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.
A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.
A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.
A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.
A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.
A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.
A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.
A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.
A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.
A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.
A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.
A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.
A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.
A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.
A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.

from contextlib import nullcontext
import torch
from transformers import AlbertTokenizer, AlbertModel
from sklearn.manifold import TSNE
from sklearn.cluster import KMeans
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from tqdm import tqdm
from torch.utils.data import DataLoader, Dataset
from transformers import AutoTokenizer, AutoModel
from torch.cuda.amp import autocast
import os
os.environ["TOKENIZERS_PARALLELISM"] = "false"


# tokenizer = AlbertTokenizer.from_pretrained(model_name)
# model = AlbertModel.from_pretrained(model_name)


class TextDataset(Dataset):
    def __init__(self, texts, tokenizer, max_length):
        self.texts = texts
        self.tokenizer = tokenizer
        self.max_length = max_length

    def __len__(self):
        return len(self.texts)

    def __getitem__(self, idx):
        text = self.texts[idx]
        inputs = self.tokenizer(text, padding="max_length", truncation="longest_first", 
                                max_length=self.max_length, return_tensors="pt")
        return {k: v.squeeze(0) for k, v in inputs.items()}

def get_embeddings(texts, model, tokenizer=None, batch_size=32, max_length=512, use_gpu=True, model_type='auto'):
    device = torch.device('cuda' if use_gpu and torch.cuda.is_available() else 'cpu')
    model = model.to(device)
    
    if model_type == 'auto':
        model = model.half() if use_gpu else model
        model.eval()

        dataset = TextDataset(texts, tokenizer, max_length)
        dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=False, num_workers=0 if use_gpu else 4)

        embeddings = []

        with torch.no_grad(), (autocast() if use_gpu else nullcontext()):
            for batch in tqdm(dataloader, desc="Generating embeddings"):
                batch = {k: v.to(device) for k, v in batch.items()}
                outputs = model(**batch)
                batch_embeddings = outputs.last_hidden_state[:, 0, :]
                embeddings.append(batch_embeddings.cpu())

        return torch.cat(embeddings).numpy()
    
    elif model_type == 'sentence_transformer':
        if isinstance(texts, pd.Series):
            texts = texts.tolist()



        model.eval()
        if use_gpu:
            model = model.half()  # Use half-precision if on GPU

        embeddings = []
        
        with torch.no_grad(), (torch.cuda.amp.autocast() if use_gpu else nullcontext()):
            for i in tqdm(range(0, len(texts), batch_size), desc="Generating embeddings"):
                batch = texts[i:i+batch_size]
                batch_embeddings = model.encode(batch, device=device, convert_to_tensor=True)
                embeddings.append(batch_embeddings.cpu())

        return torch.cat(embeddings).numpy()
    
    else:
        raise ValueError("Invalid model_type. Choose 'auto' or 'sentence_transformer'.")



# def get_embeddings(texts, batch_size=32):
#     device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
#     model.to(device)
#     model.eval()
#     
#     embeddings = []
#     
#     for i in tqdm(range(0, len(texts), batch_size), desc="Generating embeddings"):
#         batch_texts = texts[i:i+batch_size]
#         
#         # Truncate and tokenize
#         truncated_texts = [truncate_from_middle(text, 512) for text in batch_texts]
#         inputs = tokenizer(truncated_texts, return_tensors="pt", padding=True, truncation=True, max_length=512)
#         
#         inputs = {k: v.to(device) for k, v in inputs.items()}
#         
#         with torch.no_grad():
#             outputs = model(**inputs)
#         
#         # Use the [CLS] token embedding as the sentence embedding
#         batch_embeddings = outputs.last_hidden_state[:, 0, :].cpu().numpy()
#         embeddings.extend(batch_embeddings)
#     
#     return np.array(embeddings)

def cluster_and_visualize(embeddings, n_clusters=5):
    # Perform K-means clustering
    kmeans = KMeans(n_clusters=n_clusters, random_state=42)
    cluster_labels = kmeans.fit_predict(embeddings)
    
    # Perform t-SNE for visualization
    tsne = TSNE(n_components=2, random_state=42)
    reduced_embeddings = tsne.fit_transform(embeddings)
    
    # Visualize
    plt.figure(figsize=(10, 8))
    scatter = plt.scatter(reduced_embeddings[:, 0], reduced_embeddings[:, 1], 
                          c=cluster_labels, alpha=0.6, cmap='viridis')
    plt.colorbar(scatter)
    plt.title('t-SNE visualization of text embeddings')
    plt.xlabel('t-SNE feature 1')
    plt.ylabel('t-SNE feature 2')
    plt.show()
    
    return cluster_labels

from sentence_transformers import SentenceTransformer

model_name = "huawei-noah/TinyBERT_General_4L_312D"
tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=True)
model = AutoModel.from_pretrained(model_name)
# Usage for SentenceTransformer
model_st = SentenceTransformer('all-MiniLM-L6-v2')

embedding_df = ds.full_data.copy()

embeddings_tinybert = get_embeddings(embedding_df["text"], model,  tokenizer, batch_size=256, use_gpu=True, model_type='auto')

Generating embeddings:   3%|▎         | 6/176 [00:27<13:13,  4.67s/it]

---------------------------------------------------------------------------
KeyboardInterrupt                         Traceback (most recent call last)
Cell In[67], line 3
      1 embedding_df = ds.full_data.copy()
----> 3 embeddings_tinybert = get_embeddings(embedding_df["text"], model,  tokenizer, batch_size=256, use_gpu=True, model_type='auto')

Cell In[62], line 54, in get_embeddings(texts, model, tokenizer, batch_size, max_length, use_gpu, model_type)
     52             outputs = model(**batch)
     53             batch_embeddings = outputs.last_hidden_state[:, 0, :]
---> 54             embeddings.append(batch_embeddings.cpu())
     56     return torch.cat(embeddings).numpy()
     58 elif model_type == 'sentence_transformer':

KeyboardInterrupt:

embeddings_st = get_embeddings(embedding_df["text"], model_st, batch_size=256, use_gpu=True, model_type='sentence_transformer')

Generating embeddings: 100%|██████████| 176/176 [04:42<00:00,  1.61s/it]

import gc
gc.collect()
torch.cuda.empty_cache()

embeddings_st

array([[-0.04292417,  0.04017529,  0.00822352, ...,  0.00830901,
         0.04013212, -0.05353491],
       [-0.0300238 , -0.05997578,  0.09334268, ..., -0.08667795,
        -0.02973704,  0.06296595],
       [ 0.0077344 ,  0.01727735,  0.01518998, ..., -0.06386147,
        -0.015207  , -0.02204838],
       ...,
       [-0.00774679, -0.00621239,  0.02480308, ..., -0.05097079,
         0.00514519,  0.00038478],
       [ 0.06024081,  0.06292548,  0.00465201, ..., -0.00634774,
         0.04170959,  0.02873757],
       [ 0.00446899, -0.08325394, -0.00282552, ..., -0.16707481,
         0.09308673, -0.04893658]], dtype=float32)

# Copyimport numpy as np
import matplotlib.pyplot as plt
from sklearn.manifold import TSNE
from hdbscan import HDBSCAN
from sklearn.metrics import silhouette_score

def cluster_and_visualize(embeddings, min_cluster_size=500, min_samples=5):
    # HDBSCAN Clustering
    clusterer = HDBSCAN(min_cluster_size=min_cluster_size, min_samples=min_samples)
    cluster_labels = clusterer.fit_predict(embeddings)

    # Calculate silhouette score
    silhouette_avg = silhouette_score(embeddings, cluster_labels)
    print(f"Silhouette Score: {silhouette_avg}")

    # t-SNE for dimensionality reduction
    tsne = TSNE(n_components=2, random_state=42)
    embeddings_2d = tsne.fit_transform(embeddings)

    # Plotting
    plt.figure(figsize=(10, 8))
    scatter = plt.scatter(embeddings_2d[:, 0], embeddings_2d[:, 1], c=cluster_labels, cmap='viridis')
    plt.colorbar(scatter)
    plt.title(f'HDBSCAN Clustering of Text Embeddings (Silhouette Score: {silhouette_avg:.2f})')
    plt.xlabel('t-SNE feature 0')
    plt.ylabel('t-SNE feature 1')
    plt.show()

    return cluster_labels

cluster_labels = cluster_and_visualize(embeddings_st)

Pipeline options:

1. Use spacy to summarize?

importlib.reload(da_utils)
da_utils.sample_plot_by_time(ds.full_data)

<Figure size 1900x1100 with 0 Axes>

THRESHOLD = 0.25
dropped_cols = d_sorted[d_sorted['prop_fake_normalized'].abs() > THRESHOLD]
print(f"Dropping total columns: {len(dropped_cols)}")
drop_cols = dropped_cols["col"]
dropped_cols.sort_values(by='total', ascending=False)

Dropping total columns: 90

from sklearn.model_selection import train_test_split

numeric_and_binary = ds.full_data.select_dtypes(include=[np.number, 'bool'])
X = numeric_and_binary.drop('fake', axis=1)  # All columns except 'fake'
y = numeric_and_binary['fake']  # Target column

# Split the data
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

X_2 = X.drop(columns=drop_cols)
X_train_no_r, X_test_no_r, y_train_no_r, y_test_no_r = train_test_split(X_2, y, test_size=0.2, random_state=42)

print(len(X.columns))
print(len(X_train_no_r.columns))

282
192

importlib.reload(ensemble_wrapper)
tuning_results, tuning_params = ensemble_wrapper.tune_lgbm_model(
    X_train=X_train,
    y_train=y_train,
    X_val=X_test,
    y_val=y_test,
    n_trials=3, verbose=2)

tuning_results_no_r, tuning_params_no_r = ensemble_wrapper.tune_lgbm_model(
    X_train=X_train_no_r,
    y_train=y_train_no_r,
    X_val=X_test_no_r,
    y_val=y_test_no_r,
    n_trials=3, verbose=2)

tuning_results

best_params = {'num_leaves': 105,
  'learning_rate': 0.711447600934342,
  'feature_fraction': 0.7587945476302645,
  'bagging_fraction': 0.6387926357773329,
  'bagging_freq': 2,
  'min_child_samples': 19,
  'lambda_l1': 3.3323645788192616e-08,
  'lambda_l2': 0.6245760287469893,
  'value': 0.9996621866873512}

# final_model, _eval_results = ensemble_wrapper.train_lgbm_model(X_train, y_train, X_test, y_test, best_params)
final_model_no_r, _eval_results_no_r = ensemble_wrapper.train_lgbm_model(X_train_no_r, y_train_no_r, X_test_no_r, y_test_no_r, best_params)

Training until validation scores don't improve for 50 rounds
[100]	valid_0's auc: 0.9976
[200]	valid_0's auc: 0.997842
[300]	valid_0's auc: 0.997924
[400]	valid_0's auc: 0.99803
Early stopping, best iteration is:
[402]	valid_0's auc: 0.998034
Training LightGBM model took 7.13 seconds

test_metrics = ensemble_wrapper.evaluate_model(final_model_no_r, X_test_no_r, y_test_no_r)
print("Test Metrics:")
test_metrics

Test Metrics:

{'accuracy': 0.9816258351893096,
 'precision': 0.9840255591054313,
 'recall': 0.9808917197452229,
 'f1': 0.9824561403508771,
 'auc': 0.998033632164362,
 'log_loss': 0.08163107432287664}

importlib.reload(shap)
explainer, shap_values = shap.plot_shap_summary(final_model_no_r, X_test_no_r, plot_type="beeswarm")

/usr/local/lib/python3.10/dist-packages/shap/explainers/_tree.py:448: UserWarning: LightGBM binary classifier with TreeExplainer shap values output has changed to a list of ndarray
  warnings.warn('LightGBM binary classifier with TreeExplainer shap values output has changed to a list of ndarray')

if VERBOSE:
    importlib.reload(ensemble_wrapper)
    combined_imp = ensemble_wrapper.get_combined_feature_importance(final_model_no_r, X_train_no_r)
    combined_imp_test = ensemble_wrapper.get_combined_feature_importance(final_model_no_r, X_test_no_r)

    # feature_imp_df = ensemble_wrapper.get_feature_importances(final_model_no_r, X_test_no_r.columns)
    # display(feature_imp_df.sort_values(by=["importance_percentage"], ascending=False).head(25))

/usr/local/lib/python3.10/dist-packages/shap/explainers/_tree.py:448: UserWarning: LightGBM binary classifier with TreeExplainer shap values output has changed to a list of ndarray
  warnings.warn('LightGBM binary classifier with TreeExplainer shap values output has changed to a list of ndarray')
/usr/local/lib/python3.10/dist-packages/shap/explainers/_tree.py:448: UserWarning: LightGBM binary classifier with TreeExplainer shap values output has changed to a list of ndarray
  warnings.warn('LightGBM binary classifier with TreeExplainer shap values output has changed to a list of ndarray')

if VERBOSE:
    display(combined_imp.sort_values(by=["mean_abs_shap"], ascending=False).head(25))

# Plot SHAP dependence for a specific feature
shap.plot_shap_dependence(final_model_no_r, X_test_no_r, "pos_ratio_PUNCT")

/usr/local/lib/python3.10/dist-packages/shap/explainers/_tree.py:448: UserWarning: LightGBM binary classifier with TreeExplainer shap values output has changed to a list of ndarray
  warnings.warn('LightGBM binary classifier with TreeExplainer shap values output has changed to a list of ndarray')

---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
Cell In[34], line 2
      1 # Plot SHAP dependence for a specific feature
----> 2 shap.plot_shap_dependence(final_model_no_r, X_test_no_r, "pos_ratio_PUNCT")

File /mnt/v/projects/DL_s4/src/models/shared/shap_analysis.py:67, in plot_shap_dependence(model, X, feature)
     64 shap_values = explainer.shap_values(X)
     66 plt.figure(figsize=(10, 6))
---> 67 shap.dependence_plot(feature, shap_values[1], X, show=False)
     68 plt.title(f"SHAP Dependence Plot for {feature}")
     69 plt.tight_layout()

File /usr/local/lib/python3.10/dist-packages/shap/plots/_scatter.py:572, in dependence_legacy(ind, shap_values, features, feature_names, display_features, interaction_index, color, axis_color, cmap, dot_size, x_jitter, alpha, title, xmin, xmax, ax, show, ymin, ymax)
    570 if not hasattr(ind, "__len__"):
    571     if interaction_index == "auto":
--> 572         interaction_index = approximate_interactions(ind, shap_values, features)[0]
    573     interaction_index = convert_name(interaction_index, shap_values, feature_names)
    574 categorical_interaction = False

File /usr/local/lib/python3.10/dist-packages/shap/utils/_general.py:123, in approximate_interactions(index, shap_values, X, feature_names)
    121 x = X[inds, index]
    122 srt = np.argsort(x)
--> 123 shap_ref = shap_values[inds, index]
    124 shap_ref = shap_ref[srt]
    125 inc = max(min(int(len(x) / 10.0), 50), 1)

IndexError: index 37 is out of bounds for axis 1 with size 1

<Figure size 1000x600 with 0 Axes>

da_utils.render_rel_kde_plot(ds.full_data, 'pos_ratio_PUNCT')

importlib.reload(da_utils)
da_utils.corr_strip(ds.full_data, 'pos_ratio_PUNCT')

da_utils.render_rel_kde_plot(ds.full_data, 'ner_density')

importlib.reload(da_utils)
da_utils.corr_strip(ds.full_data, 'ner_density')

da_utils.render_rel_kde_plot(ds.full_data, 'verb_tense_ratio_past', x_lim=(-0.1,1.1))
da_utils.corr_strip(ds.full_data, 'verb_tense_ratio_past')

da_utils.render_rel_kde_plot(ds.full_data, 'avg_token_vector_norm', x_lim=(40, 80))
da_utils.corr_strip(ds.full_data, 'avg_token_vector_norm')

da_utils.render_rel_kde_plot(ds.full_data, 'punctuation_density', x_lim=(-0.01, 0.1))
da_utils.corr_strip(ds.full_data, 'punctuation_density')

da_utils.render_rel_kde_plot(ds.full_data, 'std_sent_length', x_lim=(-1, 45))
da_utils.corr_strip(ds.full_data, 'std_sent_length')

da_utils.render_rel_kde_plot(ds.full_data, 'pos_ratio_ADV', x_lim=(-0.01, 0.1))
da_utils.corr_strip(ds.full_data, 'pos_ratio_ADV')

from types import SimpleNamespace
import matplotlib.pyplot as plt
import seaborn as sns
import src.ds_utils.topic_analysis as topic_analysis
import src.ds_utils.da_utils as da_utils
import src.models.clustering.cluster as cluster
import src.preprocessing.pipeline as pipeline
import importlib
import configs
import pandas as pd
import numpy as np
from sklearn.metrics import normalized_mutual_info_score, adjusted_rand_score
from wordcloud import WordCloud
from matplotlib.colors import Normalize
from mpl_toolkits.axes_grid1 import make_axes_locatable

# Configuration

importlib.reload(configs)

target_embedding_columns = configs.target_embedding_columns
sematic_analysis_cols = configs.sematic_analysis_cols
sub_analysis_cols = configs.sub_analysis_cols


sns.set_theme(style="darkgrid", palette="pastel")
plt.style.use("fivethirtyeight")

VERBOSE = False

# Data Loading and various init steps
ds = pipeline.ProcessedDataset('ds_parquet_cache/ds_tokens_512_44246_bert_50_train_n_35838')
train_df = ds.train
entity_cols = [col for col in train_df.columns if col.startswith(('org_', 'gpe_', 'person_'))]
train_df['date'] = pd.to_datetime(train_df['date'], format='mixed', errors='coerce')
clean_df = train_df.dropna(subset=['date']).reset_index(drop=True)

if VERBOSE:
    unparsable = train_df['date'].isna().sum()
    total = len(train_df)
    print(f"Broken dates: {unparsable}/{total}")

['person_Trump',
 'person_Obama',
 'person_Clinton',
 'person_Hillary',
 'person_Sanders',
 'person_Cruz',
 'person_Putin',
 'person_Ryan',
 'person_Bush',
 'person_McCain',
 'person_@realDonaldTrump',
 'person_Rubio',
 'person_Donald Trump s',
 'person_Flynn',
 'person_Obamacare',
 'person_Comey',
 'person_Merkel',
 'person_Pence',
 'person_Moore',
 'person_McConnell',
 'person_Tillerson',
 'person_Johnson',
 'person_Kelly',
 'person_Assad',
 'person_Carson',
 'person_Hillary Clinton s',
 'person_Bannon',
 'person_Warren',
 'person_Donald',
 'person_Conway',
 'person_Jones',
 'person_Sean Spicer',
 'person_Hariri',
 'person_Mitt Romney',
 'person_Erdogan',
 'person_Biden',
 'person_Brown',
 'person_Xi',
 'person_Graham',
 'person_Scalia',
 'person_Schumer',
 'person_Palin',
 'person_Soros',
 'person_Mattis',
 'person_Kushner',
 'person_Mnuchin',
 'person_Manafort',
 'person_Macron',
 'person_Robert Mueller',
 'person_Reagan',
 'org_the White House',
 'org_Senate',
 'org_Congress',
 'org_FBI',
 'org_State',
 'org_CNN',
 'org_EU',
 'org_GOP',
 'org_the State Department',
 'org_the Supreme Court',
 'org_Fox News',
 'org_Facebook',
 'org_the New York Times',
 'org_U.N.',
 'org_the Washington Post',
 'org_CIA',
 'org_ISIS',
 'org_Twitter',
 'org_the Republican Party',
 'org_NATO',
 'org_the Democratic Party',
 'org_the United Nations',
 'org_DNC',
 'org_Comey',
 'org_the House of Representatives',
 'org_the European Union',
 'org_Commission',
 'org_MSNBC',
 'org_NBC',
 'org_Army',
 'org_Justice Department',
 'org_Pentagon',
 'org_Fox',
 'org_EPA',
 'org_Breitbart News',
 'org_the Clinton Foundation',
 'org_Planned Parenthood',
 'org_U.N. Security Council',
 'org_Treasury',
 'org_the Department of Justice',
 'org_al Qaeda',
 'org_Islam',
 'org_Hezbollah',
 'org_Kremlin',
 'org_Medicaid',
 'org_ABC News',
 'org_Capitol Hill',
 'org_Politico',
 'org_NFL',
 'org_Google',
 'gpe_U.S.',
 'gpe_the United States',
 'gpe_Russia',
 'gpe_America',
 'gpe_Trump',
 'gpe_China',
 'gpe_Washington',
 'gpe_Iran',
 'gpe_North Korea',
 'gpe_Syria',
 'gpe_New York',
 'gpe_Iraq',
 'gpe_Mexico',
 'gpe_Israel',
 'gpe_Britain',
 'gpe_Texas',
 'gpe_Florida',
 'gpe_Germany',
 'gpe_Turkey',
 'gpe_California',
 'gpe_Moscow',
 'gpe_Myanmar',
 'gpe_Saudi Arabia',
 'gpe_North Carolina',
 'gpe_Japan',
 'gpe_France',
 'gpe_Virginia',
 'gpe_India',
 'gpe_Afghanistan',
 'gpe_Cuba',
 'gpe_Ohio',
 'gpe_Beijing',
 'gpe_London',
 'gpe_Ukraine',
 'gpe_Jerusalem',
 'gpe_Paris',
 'gpe_Michigan',
 'gpe_D.C.',
 'gpe_Yemen',
 'gpe_Chicago',
 'gpe_Libya',
 'gpe_Canada',
 'gpe_Australia',
 'gpe_Iowa',
 'gpe_Pakistan',
 'gpe_Pyongyang',
 'gpe_Arizona',
 'gpe_Hollywood',
 'gpe_Puerto Rico',
 'gpe_UK']

plt.figure(figsize=(11, 6))

sns.histplot(
    clean_df,
    x="date",
    hue="fake",
    multiple="stack",
    edgecolor=".3",
    linewidth=.5,
    log_scale=False,
)

plt.title("Distribution of Articles Over Time")
plt.tight_layout()
plt.show()

plt.figure(figsize=(11, 6))

sns.histplot(
    clean_df,
    x="date",
    hue="subject",
    multiple="stack",
    edgecolor=".3",
    linewidth=.5,
    log_scale=False,
)

plt.title("Distribution of Articles Over Time by Subject")
plt.tight_layout()
plt.show()

validation_table = train_df.groupby("subject")["fake"].agg(['mean', 'count'])
validation_table.columns = ['Fake_Ratio', 'Total_Count']
validation_table

importlib.reload(da_utils)
sig_cors_1 = da_utils.corr_plot(clean_df[(["fake"] + sematic_analysis_cols)], target_var="fake")

sig_cors_1

sig_cors_2 = da_utils.corr_plot(clean_df[(["fake"] + sub_analysis_cols)], target_var="fake")

sig_cors_2

### { TODO: show basic XGBBOst model results with these variables}

# results:
# TODO FIX: I THINK THIS INCLUDES ENTITY COLUMNS, CHECK
classifier_report = pd.DataFrame(data = {
    "": ["precision", "recall", "f1-score", "support"],
    "0": [0.972950, 0.975476, 0.974211, 3466.000000],
    "1": [0.976971, 0.974595, 0.975781, 3700.000000],
    "accuracy": [0.975021, 0.975021, 0.975021, 0.975021],
    "macro avg": [0.974960, 0.975035, 0.974996, 7166.000000],
    "weighted avg": [0.975026, 0.975021, 0.975022, 7166.000000]
})
classifier_report.set_index("", inplace=True)
classifier_report

# TODO: use this to show feature importances for model
# importlib.reload(xgboost_utils)
#
# plots = xgboost_utils.generate_xgboost_evaluation_plots(results, top_n_features=15, max_plots=20)
# # fig = plots.feature_importance()
# # fig.show()
#
# fig = plots.feature_distribution()
# fig.show()

print(len(entity_cols))

150

importlib.reload(da_utils)
clean_df['date'] = pd.to_datetime(clean_df['date'], errors='coerce')

entity_prefixes = ['person_', 'org_', 'gpe_']
all_ratios = pd.concat([da_utils.prepare_entity_data(clean_df, prefix) for prefix in entity_prefixes], axis=1)

plot_popular = da_utils.plot_entity_popularity(all_ratios, "Most Popular Entities Over Time")

popular_entities = set(all_ratios.mean().nlargest(9).index)

variability = all_ratios.apply(da_utils.calculate_variability)

top_variable_entities = variability[~variability.index.isin(popular_entities)].nlargest(9).index

plot_variable = da_utils.plot_entity_popularity(all_ratios, "Entities with Most Significant Changes Over Time",
                                       exclude_entities=popular_entities)

plot_popular.show()
plot_variable.show()

/tmp/ipykernel_5925/2659568793.py:18: UserWarning: FigureCanvasAgg is non-interactive, and thus cannot be shown
  plot_popular.show()
/tmp/ipykernel_5925/2659568793.py:19: UserWarning: FigureCanvasAgg is non-interactive, and thus cannot be shown
  plot_variable.show()

all_ratios

# TODO: instead render separate LINES for fake and non-fake

all_ratios = pd.concat([da_utils.prepare_entity_data(clean_df[clean_df["fake"] == 1], prefix) for prefix in entity_prefixes], axis=1)


importlib.reload(da_utils)
gpe_ents = [item for item in all_ratios.mean().nlargest(100).index if item[1] == "gpe" and item[0] != "Trump"]
gpe_ents = gpe_ents[:18]

all_entities = set(all_ratios.columns)
gpe_ents_set = set(gpe_ents)
exclude_entities = all_entities - gpe_ents_set


plot_variable = da_utils.plot_entity_popularity(all_ratios, "Most popular 'gpe' Entities (only 'Fake')",top_n=18,
                                       exclude_entities=exclude_entities)

importlib.reload(da_utils)
org_ents = [item for item in all_ratios.mean().nlargest(100).index if item[1] == "org" and item[0] != "Trump"]
org_ents = org_ents[:18]

all_entities = set(all_ratios.columns)
org_ents_set = set(org_ents)
exclude_entities = all_entities - org_ents_set

plot_variable = da_utils.plot_entity_popularity(all_ratios, "Most popular 'org' Entities (only 'Fake')",top_n=18,
                                       exclude_entities=exclude_entities)

importlib.reload(da_utils)
da_utils.show_entity_target_ratios(clean_df, target="fake", entity_cols=entity_cols)

importlib.reload(da_utils)
da_utils.sentiment_plot(clean_df, entity_cols=entity_cols)

result = (clean_df.groupby("fake")["DROPPED_Reuters"]
          .agg(**{"Reuters %": "mean", "N Reuters articles": "sum"})
          .assign(**{"N non-Reuters articles": lambda x: clean_df.groupby("fake").size() - x["N Reuters articles"]}))
result

selected_clustering_runs = SimpleNamespace(
    raw_text=SimpleNamespace(cluster_2='bmc3wnu9', multi_cluster="0180in2r"),
    processed_text=SimpleNamespace(cluster_2='8zzg1lu6', multi_cluster="17ih00oj"),
    anonymized_embeddings=SimpleNamespace(cluster_2='1rxktps6', multi_cluster="0wpy8r7p"),
)
train_df_2_cluster, metadata_source_text = cluster.apply_clustering_to_df(train_df, target_embedding_columns.raw_text,
                                                                          'bmc3wnu9')
train_df_2_cluster, metadata_processed_text = cluster.apply_clustering_to_df(train_df_2_cluster,
                                                                             target_embedding_columns.processed_text,
                                                                             '8zzg1lu6')
train_df_2_cluster, metadata_anonymized_text = cluster.apply_clustering_to_df(train_df_2_cluster,
                                                                              target_embedding_columns.anonymized_embeddings,
                                                                              '1rxktps6')
train_df_multi_cluster, metadata_multi_source_text = cluster.apply_clustering_to_df(train_df,
                                                                                    target_embedding_columns.raw_text,
                                                                                    selected_clustering_runs.raw_text.multi_cluster)
train_df_multi_cluster, metadata_multi_processed_text = cluster.apply_clustering_to_df(train_df_multi_cluster,
                                                                                       target_embedding_columns.processed_text,
                                                                                       selected_clustering_runs.processed_text.multi_cluster)
train_df_multi_cluster, metadata_multi_anonymized_text = cluster.apply_clustering_to_df(train_df_multi_cluster,
                                                                                        target_embedding_columns.anonymized_embeddings,
                                                                                        selected_clustering_runs.anonymized_embeddings.multi_cluster)

def get_tsne_for_embeddings(embeddings, perplexity=30, n_iter=1000):
    scaler = StandardScaler()
    scaled_features = scaler.fit_transform(embeddings)

    tsne = TSNE(n_components=2, random_state=42, perplexity=perplexity, n_iter=n_iter)
    tsne_results = tsne.fit_transform(scaled_features)
    return tsne_results


# Generate t-SNE embeddings
tsne_2d_source = get_tsne_for_embeddings(np.stack(train_df[target_embedding_columns.raw_text.embedding].values))
tsne_2d_processed = get_tsne_for_embeddings(
    np.stack(train_df[target_embedding_columns.processed_text.embedding].values))
tsne_2d_anonymized = get_tsne_for_embeddings(
    np.stack(train_df[target_embedding_columns.anonymized_embeddings.embedding].values))

---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
Cell In[22], line 34
     30     return tsne_results
     33 # Generate t-SNE embeddings
---> 34 tsne_2d_source = get_tsne_for_embeddings(np.stack(train_df[target_embedding_columns.raw_text.embedding].values))
     35 tsne_2d_processed = get_tsne_for_embeddings(
     36     np.stack(train_df[target_embedding_columns.processed_text.embedding].values))
     37 tsne_2d_anonymized = get_tsne_for_embeddings(
     38     np.stack(train_df[target_embedding_columns.anonymized_embeddings.embedding].values))

Cell In[22], line 25, in get_tsne_for_embeddings(embeddings, perplexity, n_iter)
     24 def get_tsne_for_embeddings(embeddings, perplexity=30, n_iter=1000):
---> 25     scaler = StandardScaler()
     26     scaled_features = scaler.fit_transform(embeddings)
     28     tsne = TSNE(n_components=2, random_state=42, perplexity=perplexity, n_iter=n_iter)

NameError: name 'StandardScaler' is not defined

# Search Grid
{
            # UMAP parameters
            'n_neighbors': {'values': [15, 30, 60]},
            'min_dist': {'values': [0.1]},
            'n_components': {'values': [30]},

            #  HDBSCAN parameters
            'min_cluster_size': {'values': [1000]},
            'min_samples': {'values': [100, 250, 500]},
            'cluster_selection_epsilon': {'values': [0.0, 0.25, 0.5]},
            'cluster_selection_method': {'values': ['eom', 'leaf']},
            'alpha': {'values': [0.5, 1.0, 1.5]},
        }

{TODO: Add WANDB EMBEDED PAGE} {TODO: check last project}

# target_embedding_columns.raw_text: 
# Best 2 Cluster
# bmc3wnu9 - 

# Best 3  clusters
# 0ib21i84 - 

# Best 5  clusters
# 0180in2r -


# target_embedding_columns.processed_text

# Best 2 Cluster
#  8zzg1lu6 - 

# Best 3  clusters
#  6dkf6d5x - 

# Best 4  clusters
#  17ih00oj -

# target_embedding_columns.anonymized_text

# Best 2 Cluster
#  1rxktps6 - 

# Best 3  clusters
#  do6sv3xg - 

# Best 4  clusters
# 0wpy8r7p  -

importlib.reload(da_utils)
da_utils.plot_embedding_grid([tsne_2d_source, tsne_2d_processed, tsne_2d_anonymized],
                             [train_df["fake"], train_df["fake"], train_df["fake"]],
                             titles=['Source Text', 'Processed Text', 'Anonymized Text'],
                             embedding_type="t-SNE", top_level_title="t-SNE by 'fake' (target variable")

importlib.reload(da_utils)
da_utils.plot_embedding_grid([tsne_2d_source, tsne_2d_processed, tsne_2d_anonymized],
                             [
                                 train_df_2_cluster[f"{target_embedding_columns.raw_text.embedding}_cluster"],
                                 train_df_2_cluster[f"{target_embedding_columns.processed_text.embedding}_cluster"],
                                 train_df_2_cluster[
                                     f"{target_embedding_columns.anonymized_embeddings.embedding}_cluster"]
                             ],
                             titles=['Source Text', 'Processed Text', 'Anonymized Text'],
                             embedding_type="t-SNE", top_level_title="t-SNE by UMAP + HDBSCAN cluster")

importlib.reload(da_utils)
da_utils.plot_embedding_grid(None, None,
                             titles=['Source Text', 'Processed Text', 'Anonymized Text'],

                             embedding_type="UMAP",
                             dataset_hashes=[
                                 target_embedding_columns.raw_text,
                                 target_embedding_columns.processed_text,
                                 target_embedding_columns.anonymized_embeddings],
                             run_ids=[selected_clustering_runs.raw_text.cluster_2,
                                      selected_clustering_runs.processed_text.cluster_2,
                                      selected_clustering_runs.anonymized_embeddings.cluster_2],
                             top_level_title="UMAP Embeddings Comparison")

importlib.reload(da_utils)

def compare_clusters_with_target(cluster_columns, target, column_names):
    """
    Compare multiple independent HDBSCAN clustering results with a target variable.
    
    :param cluster_columns: List of Series, each containing HDBSCAN cluster labels
    :param target: Series containing the target variable
    :param column_names: List of names for each clustering column
    :return: DataFrame with comparison metrics
    """
    results = []

    for clusters, name in zip(cluster_columns, column_names):
        results.append({
            'Text Representation': name,
            'Num Clusters': len(np.unique(clusters)),
            'NMI': normalized_mutual_info_score(target, clusters),
            'ARI': adjusted_rand_score(target, clusters)
        })

    return pd.DataFrame(results)


cluster_columns = [
    train_df_2_cluster[f"{target_embedding_columns.raw_text.embedding}_cluster"],
    train_df_2_cluster[f"{target_embedding_columns.processed_text.embedding}_cluster"],
    train_df_2_cluster[f"{target_embedding_columns.anonymized_embeddings.embedding}_cluster"]
]
target = train_df["fake"]
column_names = ['Raw Text', 'Processed Text', 'Anonymized Text']

comparison_df = compare_clusters_with_target(cluster_columns, target, column_names)
comparison_df

import pandas as pd
import numpy as np
from sklearn.metrics import f1_score, precision_score, recall_score


def analyze_cluster_target_overlap(cluster_column, target):
    df = pd.DataFrame({'Cluster': cluster_column, 'Target': target})

    results = []
    for c_id in df['Cluster'].unique():
        cluster_mask = df['Cluster'] == c_id
        cluster_data = df[cluster_mask]
        cluster_size = len(cluster_data)
        fake_count = (cluster_data['Target'] == 1).sum()
        fake_proportion = fake_count / cluster_size

        majority_class = 1 if fake_proportion > 0.5 else 0

        # Treat the cluster as a classifier for its majority class
        y_true = (df['Target'] == majority_class).astype(int)
        y_pred = cluster_mask.astype(int)

        precision = precision_score(y_true, y_pred)
        recall = recall_score(y_true, y_pred)
        f1 = f1_score(y_true, y_pred)

        results.append({
            'Cluster': c_id,
            'Size': cluster_size,
            'Fake_Count': fake_count,
            'Real_Count': cluster_size - fake_count,
            'Fake_Proportion': fake_proportion,
            'Real_Proportion': 1 - fake_proportion,
            'Majority_Class': majority_class,
            'Precision': precision,
            'Recall': recall,
            'F1': f1
        })

    results_df = pd.DataFrame(results).sort_values('Size', ascending=False)

    # Calculate overall statistics
    total_size = results_df['Size'].sum()
    total_fake = results_df['Fake_Count'].sum()
    overall_fake_proportion = total_fake / total_size

    overall_row = pd.DataFrame({
        'Cluster': ['Overall'],
        'Size': [total_size],
        'Fake_Count': [total_fake],
        'Real_Count': [total_size - total_fake],
        'Fake_Proportion': [overall_fake_proportion],
        # 'Real_Proportion': [1 - overall_fake_proportion],
        'Majority_Class': [1 if overall_fake_proportion > 0.5 else 0],
        'Precision': [np.nan],
        'Recall': [np.nan],
        'F1': [np.nan]
    })

    results_df = pd.concat([results_df, overall_row], ignore_index=True)

    return results_df


raw_text_cluster_analysis_overlap = analyze_cluster_target_overlap(
    train_df_2_cluster[f"{target_embedding_columns.raw_text.embedding}_cluster"],
    train_df["fake"]
)

processed_cluster_analysis__text_overlap = analyze_cluster_target_overlap(
    train_df_2_cluster[f"{target_embedding_columns.processed_text.embedding}_cluster"],
    train_df["fake"]
)

anonymized_cluster_analysis__text_overlap = analyze_cluster_target_overlap(
    train_df_2_cluster[f"{target_embedding_columns.anonymized_embeddings.embedding}_cluster"],
    train_df["fake"]
)
# raw_text_cluster_analysis_overlap

raw_text_cluster_analysis_overlap

processed_cluster_analysis__text_overlap

anonymized_cluster_analysis__text_overlap

importlib.reload(da_utils)

da_utils.plot_embedding_grid(None, None,
                             titles=['Source Text', 'Processed Text', 'Anonymized Text'],

                             embedding_type="UMAP",
                             dataset_hashes=[
                                 target_embedding_columns.raw_text,
                                 target_embedding_columns.processed_text,
                                 target_embedding_columns.anonymized_embeddings],
                             run_ids=[selected_clustering_runs.raw_text.multi_cluster,
                                      selected_clustering_runs.processed_text.multi_cluster,
                                      selected_clustering_runs.anonymized_embeddings.multi_cluster],
                             top_level_title="UMAP Embeddings Comparison", target_df=train_df_multi_cluster)

# import src.ds_utils.topic_analysis as topic_analysis
# 
# importlib.reload(topic_analysis)
# 
# cluster_column = f"{target_embedding_columns.raw_text.embedding}_cluster"
# 
# word_clouds = topic_analysis.analyze_clusters(
#     train_df_multi_cluster.sample(500, random_state=42).reset_index(drop=True),
#     text_column='text',
#     cluster_column=cluster_column)
# # for cluster, fig in word_clouds.items():
# #     fig.write_html(f'cluster_{cluster}_word_cloud.html')

# for cluster, fig in word_clouds.items():
#     fig.write_html(f'IGNORE_cluster_{cluster}_word_cloud.html')

# Function to merge topics for each row
def merge_topics(row):
    active_entities = [col for col in entity_cols if bool(row[col])]

    row_topics = {}

    for topic, data in topic_analysis.MERGED_TOPICS.items():
        topic_key = f"topic_{topic}"
        if set(active_entities).intersection(data["tags"]):
            row_topics[topic_key] = 1
        else:
            row_topics[topic_key] = 0

    return pd.Series(row_topics)


topics_usage_df = train_df_multi_cluster.apply(merge_topics, axis=1)

# # anonymized_embeddings processed_text raw_text
# for c in [f"{target_embedding_columns.anonymized_embeddings.embedding}_cluster",
#           f"{target_embedding_columns.processed_text.embedding}_cluster",
#           f"{target_embedding_columns.raw_text.embedding}_cluster"]:
#     topics_usage_df[c] = train_df_multi_cluster[c]

importlib.reload(topic_analysis)



def generate_entity_wordclouds(df, cluster_column, target_column="fake",
                               entity_prefix=('topic_', 'org_', 'gpe_', 'person_'),
                               top_n=50, distinctiveness_weight=0.5):
    entity_cols = [col for col in df.columns if col.startswith(entity_prefix)]

    grouped = df.groupby(cluster_column)[entity_cols].sum()

    total_occurrences = grouped.sum()
    relative_freq = grouped.div(grouped.sum(axis=1), axis=0)

    avg_relative_freq = relative_freq.mean()

    distinctiveness = relative_freq.div(avg_relative_freq) - 1

    entity_target_props = {}
    for entity in entity_cols:
        entity_mask = df[entity] > 0
        entity_target_props[entity] = df.loc[entity_mask, target_column].mean()

    wordclouds = {}
    scores = {}
    topic_scores = {}

    for cluster in grouped.index:
        cluster_scores = {}
        cluster_topic_scores = {}
        for entity in entity_cols:
            abs_freq = grouped.loc[cluster, entity]
            rel_freq = relative_freq.loc[cluster, entity]
            dist_score = distinctiveness.loc[cluster, entity]

            score = (1 - distinctiveness_weight) * rel_freq + distinctiveness_weight * max(dist_score, 0)

            if entity.startswith("topic_"):
                cluster_topic_scores[entity.split('_', 1)[1]] = score
                continue

            if score > 0:
                cluster_scores[entity.split('_', 1)[1]] = score

        topic_scores[cluster] = cluster_topic_scores
        max_score = max(cluster_scores.values()) if cluster_scores else 1
        normalized_scores = {k: v / max_score for k, v in cluster_scores.items()}

        top_entities = dict(sorted(normalized_scores.items(), key=lambda x: x[1], reverse=True)[:top_n])

        palette = sns.color_palette("RdBu_r", as_cmap=True)

        def color_func(word, font_size, position, orientation, random_state=None, **kwargs):
            entity_key = next((key for key in entity_target_props if key.endswith(word)), None)
            if entity_key:
                value = entity_target_props[entity_key]
                rgb = palette(value)[:3]  # Get RGB values from the colormap
                return f"rgb({int(rgb[0] * 255)}, {int(rgb[1] * 255)}, {int(rgb[2] * 255)})"
            return "grey"

        wordclouds[cluster] = WordCloud(width=1200, height=600, background_color='white').generate_from_frequencies(
            top_entities)
        wordclouds[cluster] = wordclouds[cluster].recolor(color_func=color_func, random_state=42)

        scores[cluster] = top_entities

    return wordclouds, scores, topic_scores, entity_target_props


def plot_wordcloud(wordcloud, title, topic_scores, entity_target_props):
    fig, ax = plt.subplots(figsize=(12, 6))

    img = ax.imshow(wordcloud, interpolation='bilinear')
    ax.axis('off')

    divider = make_axes_locatable(ax)
    cax = divider.append_axes("right", size="5%", pad=0.1)

    cmap = sns.color_palette("RdBu_r", as_cmap=True)

    norm = Normalize(vmin=0, vmax=1)
    sm = plt.cm.ScalarMappable(cmap=cmap, norm=norm)
    sm.set_array([])

    cbar = plt.colorbar(sm, cax=cax)
    cbar.set_label('Usage in Fake (%)')

    top_2_keys = sorted(topic_scores, key=topic_scores.get, reverse=True)[:2]
    top_2_keys = ", ".join([k.replace("_", " ") for k in top_2_keys])
    ax.set_title(f"{title}: {top_2_keys}", loc='left', pad=20, fontsize=15)

    plt.tight_layout(pad=0)
    plt.show()


wordclouds, scores, topic_scores, entity_target_props = generate_entity_wordclouds(
    pd.concat([train_df_multi_cluster.reset_index(drop=True), topics_usage_df.reset_index(drop=True)], axis=1),
    f"{target_embedding_columns.processed_text.embedding}_cluster")
# wordclouds, scores = generate_entity_wordclouds(train_df_multi_cluster,
#                                                 f"{target_embedding_columns.processed_text.embedding}_cluster")
for cluster, wc in wordclouds.items():
    plot_wordcloud(wc, f'Cluster {cluster} ', topic_scores[cluster], entity_target_props)


# TODO:
# CONSIDER COLORING TEXT BY PROPORTION OF "FAKE" for given STR OR SOMETHING SIMILR

# wordclouds, scores = generate_entity_wordclouds(train_df_2_cluster, f"{target_embedding_columns.processed_text.embedding}_cluster")
# wordclouds

# for cluster, wc in wordclouds.items():
#     plot_wordcloud(wc, f'Cluster {cluster} Entity Word Cloud')

# wordclouds, scores = generate_entity_wordclouds(topics_usage_df,
#                                                 f"{target_embedding_columns.processed_text.embedding}_cluster",
#
#                                                 entity_prefix=('topic_'))
# for cluster, wc in wordclouds.items():
#     plot_wordcloud(wc, f'Cluster {cluster} Entity Word Cloud')

import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sns
import importlib
from types import SimpleNamespace

import Notebooks.utils.utils as utils
import Notebooks.utils.da_utils as da_utils
import src.ds_utils.dataset as dataset

import src.models.boost_ensemble.ensemble_wrapper as ensemble_wrapper
import src.models.shared.shap_analysis as shap
import src.preprocessing.pipeline as pipeline

utils.fix_cwd()

VERBOSE =True

importlib.reload(pipeline)
# ds = pipeline.ProcessedDataset('ds_parquet_cache/ds_tokens_512_44246_bert_50_train_n_35838')
ds = pipeline.ProcessedDataset('ds_parquet_cache/ds_tokens_512_44246_bert_50_train_n_35838')
# ds = pipeline.ProcessedDataset('ds_parquet_cache/ds_tokens_512_44246_bert_50_train_n_35838')
# ds = pipeline.ProcessedDataset('ds_parquet_cache/ds_tokens_512_100_bert_50_train_n_81')
# ds = pipeline.ProcessedDataset('ds_parquet_cache/processed_dataset_44246_default_80')
df_train = ds.train

list(df_train.columns)

# if VERBOSE:
#     display(df_train[['DROPPED_Reuters', "text", "anonymized_text", "anonymized_text_bert-base-uncased_used_text"]])
df_train[['fake', 'DROPPED_Reuters', "text", "anonymized_text", "anonymized_text_bert-base-uncased_used_text"]]

[t for t in list(df_train.columns) if "source_text" in t]

['source_text',
 'source_text_bert_embedding',
 'source_text_bert-base-uncased_used_text',
 'source_text_tokens']

---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
Cell In[1], line 1
----> 1 list(df_train.columns)

NameError: name 'df_train' is not defined

target_embedding_columns = SimpleNamespace(
    anonymized_embeddings=SimpleNamespace(
        text="anonymized_text",
        embedding="anonymized_text_bert_embedding"
    ),
    processed_text=SimpleNamespace(
        text="text",
        embedding="text_bert_embedding"
    ),
    raw_text=SimpleNamespace(
        text="source_text",
        embedding="source_text_bert_embedding"
    )
)


EMBEDDING_COLUMN = "anonymized_text_bert_embedding"

df_train[target_embedding_columns.processed_text.embedding]

0        [-0.4915247, -0.67728907, -0.120532565, -0.088...
1        [-0.2743477, -0.7278832, -0.5586739, 0.0368707...
2        [-0.9693481, -0.44558665, -0.2224515, -0.60803...
3        [-0.20548154, 0.0625999, -0.05869634, 0.270725...
4        [-1.0744206, -0.2710174, -0.25682816, -0.41904...
                               ...                        
35833    [-1.0116835, -0.6445019, 0.04759822, 0.4232899...
35834    [-0.4763119, -0.48721468, -0.49852598, -0.0282...
35835    [-0.5958654, -0.012258798, -0.07052293, 0.1296...
35836    [-0.7383414, -0.25769737, -0.7854175, -1.13049...
35837    [-0.13245764, -0.7824269, 0.25529382, 0.196255...
Name: text_bert_embedding, Length: 35838, dtype: object

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.preprocessing import StandardScaler
from umap import UMAP
from hdbscan import HDBSCAN
from sklearn.metrics import silhouette_score
from tqdm.auto import tqdm
from IPython.display import display, clear_output
import numpy as np
import pandas as pd
from hdbscan import HDBSCAN
from sklearn.metrics import silhouette_score
from IPython.display import display
import umap
from joblib import Parallel, delayed
from tqdm.auto import tqdm
from tqdm.notebook import tqdm as tqdm_notebook

MIN_CLUSTER_SIZE = 400


def load_embeddings(df, column_name):
    return np.stack(df[column_name].values)


def perform_clustering(embeddings, min_samples, alpha, min_cluster_size=MIN_CLUSTER_SIZE):
    umap_model = UMAP(n_components=50, random_state=42)
    umap_embeddings = umap_model.fit_transform(StandardScaler().fit_transform(embeddings))

    hdbscan_model = HDBSCAN(min_cluster_size=min_cluster_size, 
                            min_samples=min_samples,
                            alpha=alpha,
                            cluster_selection_method='eom',
                            approx_min_span_tree=True
                           )
    labels = hdbscan_model.fit_predict(umap_embeddings)

    return labels, umap_embeddings


def tune_clustering(df, min_cluster_size=MIN_CLUSTER_SIZE, min_samples_range=range(5, 101, 5),
                    alpha_range=np.linspace(0.1, 2.5, 8)):
    df = df#.sample(8000, random_state=42)
    embeddings = load_embeddings(df, EMBEDDING_COLUMN)

    # Reduce dimensionality with progress bar
    print("Reducing dimensionality with UMAP...")
    reducer = umap.UMAP(n_components=50, random_state=42, verbose=True)
    reduced_embeddings = reducer.fit_transform(embeddings)

    def process_params(min_samples, alpha):
        hdbscan = HDBSCAN(min_cluster_size=min_cluster_size, min_samples=min_samples, alpha=alpha,
                          cluster_selection_method='eom', approx_min_span_tree=True)
        labels = hdbscan.fit_predict(reduced_embeddings)

        if len(set(labels)) > 1:
            silhouette = silhouette_score(reduced_embeddings, labels)
            n_clusters = len(set(labels)) - (1 if -1 in labels else 0)
            noise_proportion = np.sum(labels == -1) / len(labels)
            return {'min_samples': min_samples, 'alpha': alpha, 'silhouette_score': silhouette,
                    'n_clusters': n_clusters, 'noise_proportion': noise_proportion}
        return None

    print("Starting tuning process...")
    results = []
    total_iterations = len(min_samples_range) * len(alpha_range)
    
    with tqdm_notebook(total=total_iterations, desc="Tuning Progress") as pbar:
        for min_samples in min_samples_range:
            for alpha in alpha_range:
                result = process_params(min_samples, alpha)
                if result is not None:
                    results.append(result)
                pbar.update(1)

    results_df = pd.DataFrame(results)

    print("\nTuning completed. Displaying results:")
    display(results_df)

    best_params = results_df.loc[results_df['silhouette_score'].idxmax()]
    return best_params, results_df





def visualize_clusters(embeddings, labels, umap_embeddings):
    umap_2d = UMAP(n_components=2, random_state=42)
    umap_results_2d = umap_2d.fit_transform(embeddings)

    s_score = silhouette_score(umap_embeddings, labels)

    plt.figure(figsize=(10, 8))
    scatter = plt.scatter(umap_results_2d[:, 0],
                          umap_results_2d[:, 1],
                          alpha=0.75,
                          s=1,
                          c=labels,
                          cmap='Dark2')

    plt.colorbar(scatter, label='Cluster')
    plt.title(f'UMAP projection with HDBSCAN clustering n={len(labels.unique())}, S={s_score:.4f}')
    plt.xlabel('UMAP feature 1')
    plt.ylabel('UMAP feature 2')
    plt.tight_layout()
    plt.show()


def cluster_dataframe(df, embedding_column, min_samples, alpha):
    embeddings = load_embeddings(df, embedding_column)
    main_labels, umap_embeddings = perform_clustering(embeddings, min_samples=min_samples, alpha=alpha)

    df['main_cluster'] = main_labels
    df['sub_cluster'] = -1

    # for cluster in set(main_labels):
    #     if cluster != -1:
    #         cluster_mask = main_labels == cluster
    #         cluster_embeddings = umap_embeddings[cluster_mask]
    # 
    #         if len(cluster_embeddings) > 1:
    #             
    #             try:
    #                 best_params = tune_clustering(cluster_embeddings)
    #                 sub_hdbscan = HDBSCAN(min_cluster_size=50, min_samples=int(best_params['min_samples']),
    #                                       alpha=best_params['alpha'], cluster_selection_method='eom')
    #                 sub_labels = sub_hdbscan.fit_predict(cluster_embeddings)
    # 
    #                 df.loc[cluster_mask, 'sub_cluster'] = sub_labels
    #             except Exception as e:
    #                 pass
    return df, umap_embeddings

# Usage
# df_clustered = main(df_train, 'sentence_transformer_embedding')

# min_samples	alpha	silhouette_score	n_clusters	noise_proportion
 
# 149	95	1.814286	0.314928	2	0.094625
# 151	95	2.500000	0.314928	2	0.094625
# 144	95	0.100000	0.314928	2	0.094625
# 145	95	0.442857	0.314928	2	0.094625
# 146	95	0.785714	0.314928	2	0.094625

print(1)
best_params, results_df = tune_clustering(df_train)

1
Reducing dimensionality with UMAP...
UMAP(n_components=50, n_jobs=1, random_state=42, verbose=True)
Sun Aug 18 13:31:18 2024 Construct fuzzy simplicial set
Sun Aug 18 13:31:18 2024 Finding Nearest Neighbors
Sun Aug 18 13:31:18 2024 Building RP forest with 14 trees

/usr/local/lib/python3.10/dist-packages/umap/umap_.py:1945: UserWarning: n_jobs value 1 overridden to 1 by setting random_state. Use no seed for parallelism.
  warn(f"n_jobs value {self.n_jobs} overridden to 1 by setting random_state. Use no seed for parallelism.")

Sun Aug 18 13:31:20 2024 NN descent for 15 iterations

KeyboardInterrupt

# import optuna
# import numpy as np
# import pandas as pd
# from sklearn.metrics import silhouette_score, calinski_harabasz_score
# from hdbscan import HDBSCAN
# import umap
# import multiprocessing
# 
# def objective(trial, embeddings):
#     # UMAP parameters
#     n_neighbors = trial.suggest_int('n_neighbors', 30, 70)
#     min_dist = trial.suggest_float('min_dist', 0.0, 0.5)
#     n_components = trial.suggest_int('n_components', 40, 60)
#     
#     # Reduce dimensionality with UMAP
#     reducer = umap.UMAP(n_neighbors=n_neighbors, min_dist=min_dist, n_components=n_components, metric='cosine', random_state=42)
#     reduced_embeddings = reducer.fit_transform(embeddings)
#     
#     # HDBSCAN parameters
#     min_cluster_size = trial.suggest_int('min_cluster_size', 5, 50)
#     min_samples = trial.suggest_int('min_samples', 1, 20)
#     
#     # Perform clustering with HDBSCAN
#     clusterer = HDBSCAN(min_cluster_size=min_cluster_size, min_samples=min_samples, metric='euclidean', cluster_selection_method='eom')
#     labels = clusterer.fit_predict(reduced_embeddings)
#     
#     # Compute clustering metrics
#     if len(np.unique(labels)) > 1:
#         sil_score = silhouette_score(reduced_embeddings, labels)
#         ch_score = calinski_harabasz_score(reduced_embeddings, labels)
#         return sil_score  # Optimize for silhouette score
#     else:
#         return -1  # Penalize single cluster solutions
# 
# def objective_wrapper(trial, embeddings):
#     return objective(trial, embeddings)
# 
# def optimize_clustering(embeddings, n_trials=10, n_jobs=-1):
#     study = optuna.create_study(direction='maximize')
#     
#     if n_jobs == 1:
#         # Single process
#         study.optimize(lambda trial: objective(trial, embeddings), n_trials=n_trials)
#     else:
#         # Multiprocessing
#         n_jobs = multiprocessing.cpu_count() if n_jobs == -1 else n_jobs
#         with multiprocessing.Pool(processes=n_jobs) as pool:
#             futures = []
#             for _ in range(n_trials):
#                 trial = study.ask()
#                 future = pool.apply_async(objective_wrapper, (trial, embeddings))
#                 futures.append((trial, future))
#             
#             for trial, future in futures:
#                 value = future.get()
#                 study.tell(trial, value)
# 
#     print("Best trial:")
#     trial = study.best_trial
#     
#     print("Value: ", trial.value)
#     print("Params: ")
#     for key, value in trial.params.items():
#         print("    {}: {}".format(key, value))
#     
#     # Create a DataFrame with all trials
#     trials_df = study.trials_dataframe()
#     
#     return study.best_params, trials_df

import optuna
import numpy as np
import umap
import hdbscan
from sklearn.metrics import silhouette_score
import warnings
from multiprocessing import Pool, cpu_count
import contextlib

def objective(trial, embeddings):
    # UMAP parameters
    n_neighbors = trial.suggest_int('n_neighbors', 5, 50)
    min_dist = trial.suggest_float('min_dist', 0.0, 0.99)
    n_components = trial.suggest_int('n_components', 2, 100)

    # HDBSCAN parameters
    min_cluster_size = trial.suggest_int('min_cluster_size', 5, 100)
    min_samples = trial.suggest_int('min_samples', 1, 20)

    with warnings.catch_warnings():
        warnings.simplefilter("ignore")

        reducer = umap.UMAP(
            n_neighbors=n_neighbors,
            min_dist=min_dist,
            n_components=n_components,
            random_state=42
        )
        reduced_embeddings = reducer.fit_transform(embeddings)

        clusterer = hdbscan.HDBSCAN(
            min_cluster_size=min_cluster_size,
            min_samples=min_samples,
            metric='euclidean',
            cluster_selection_method='eom'
        )
        clusterer.fit(reduced_embeddings)

    labels = clusterer.labels_
    n_clusters = len(set(labels)) - (1 if -1 in labels else 0)
    noise_proportion = np.sum(labels == -1) / len(labels)

    if n_clusters > 1:
        sil_score = silhouette_score(reduced_embeddings, labels)
    else:
        sil_score = -1

    score = sil_score * (1 - noise_proportion) * n_clusters / 10
    return score

def optimize_clustering(embeddings, n_trials=100, n_jobs=None):
    n_jobs = n_jobs or cpu_count()
    
    def objective_wrapper(trial):
        return objective(trial, embeddings)

    study = optuna.create_study(direction='maximize')
    
    completed_trials = 0
    with Pool(processes=n_jobs) as pool:
        futures = []
        for _ in range(n_trials):
            trial = study.ask()
            future = pool.apply_async(objective_wrapper, (trial,))
            futures.append((trial, future))
        
        for trial, future in futures:
            try:
                result = future.get(timeout=300)  # 5 minutes timeout
                study.tell(trial, result)
                completed_trials += 1
                print(f"Completed trial {completed_trials}/{n_trials}")
            except Exception as e:
                print(f"Trial failed: {str(e)}")
                study.tell(trial, state=optuna.trial.TrialState.FAIL)

    print(f"\nOptimization completed. {completed_trials}/{n_trials} trials succeeded.")

    if completed_trials > 0:
        print("\nBest trial:")
        trial = study.best_trial
        print("  Value: ", trial.value)
        print("  Params: ")
        for key, value in trial.params.items():
            print(f"    {key}: {value}")
        return study.best_params, study.trials_dataframe()
    else:
        print("No trials completed successfully. Please check your objective function and parameters.")
        return None, study.trials_dataframe()

print(1)
best_params, results_df = optimize_clustering(load_embeddings(df_train, EMBEDDING_COLUMN))

[I 2024-08-18 13:44:42,488] A new study created in memory with name: no-name-a71888b8-3fa6-4251-b298-8763847a129c

1
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'
Trial failed: Can't pickle local object 'optimize_clustering.<locals>.objective_wrapper'

Optimization completed. 0/100 trials succeeded.
No trials completed successfully. Please check your objective function and parameters.

results_df.sort_values(by="silhouette_score", ascending=False)

df_clustered, umap_embeddings = cluster_dataframe(df_train, EMBEDDING_COLUMN, min_samples = 95, alpha=1.814286)

/usr/local/lib/python3.10/dist-packages/umap/umap_.py:1945: UserWarning: n_jobs value 1 overridden to 1 by setting random_state. Use no seed for parallelism.
  warn(f"n_jobs value {self.n_jobs} overridden to 1 by setting random_state. Use no seed for parallelism.")

visualize_clusters(
    embeddings = load_embeddings(df_clustered, EMBEDDING_COLUMN),
    labels = df_clustered['main_cluster'],
    umap_embeddings=  umap_embeddings)

/usr/local/lib/python3.10/dist-packages/umap/umap_.py:1945: UserWarning: n_jobs value 1 overridden to 1 by setting random_state. Use no seed for parallelism.
  warn(f"n_jobs value {self.n_jobs} overridden to 1 by setting random_state. Use no seed for parallelism.")

# Visualize main clustering
visualize_clusters(load_embeddings(df_clustered, 'bert_embedding'), df_clustered['main_cluster'], 'Main Clusters')

# Visualize sub-clustering for each main cluster
for cluster in set(df_clustered['main_cluster']):
    if cluster != -1:
        cluster_mask = df_clustered['main_cluster'] == cluster
        cluster_embeddings = load_embeddings(df_clustered[cluster_mask], embedding_column)
        sub_labels = df_clustered.loc[cluster_mask, 'sub_cluster']
        visualize_clusters(cluster_embeddings, sub_labels, f'Sub-clusters for Main Cluster {cluster}')

	col	total	fake	true	prop_fake
0	person_Trump	21541	11981	9560	0.556195
80	org_Reuters	21531	302	21229	0.014026
15	person_Donald Trump s	18051	8873	9178	0.491552
10	person_@realDonaldTrump	16995	7961	9034	0.468432
160	gpe_U.S.	16027	4671	11356	0.291446
...	...	...	...	...	...
61	person_Reilly	217	206	11	0.949309
53	person_Gorsuch	217	79	138	0.364055
226	gpe_Flint	201	145	56	0.721393
50	person_Mugabe	192	4	188	0.020833
140	org_SPD	179	6	173	0.033520

	total	prop_fake	importance_score
total	1.000000	-0.063512	0.388932
prop_fake	-0.063512	1.000000	-0.277533
importance_score	0.388932	-0.277533	1.000000

	fake	num_entities	num_person_entities	num_org_entities	num_gpe_entities	num_date_entities	sentiment	num_words	num_sents	avg_word_length	...	pos_ratio_PRON	pos_ratio_SPACE	pos_ratio_X	pos_ratio_SCONJ	pos_ratio_DET	pos_ratio_PUNCT	pos_ratio_CCONJ	verb_tense_ratio_past	verb_tense_ratio_present	verb_tense_ratio_future
22216	1	13	6	4	3	0	0.08333	187	7	4.58824	...	0.01070	0.02674	0.00000	0.01070	0.12299	0.05348	0.04278	0.45455	0	0
27917	0	114	49	28	18	10	0.08660	904	29	4.47898	...	0.04978	0.00221	0.00000	0.00664	0.08407	0.11394	0.02102	0.60759	0	0
25007	0	26	5	4	6	4	-0.00504	359	14	4.30084	...	0.06128	0.01114	0.00000	0.01114	0.08635	0.12813	0.01393	0.64706	0	0
1377	1	24	8	1	8	3	-0.01116	230	12	4.61739	...	0.08696	0.06957	0.00000	0.01739	0.07391	0.10870	0.01304	0.73077	0	0
32476	0	75	10	15	19	9	0.03935	606	23	4.30363	...	0.06601	0.00165	0.00000	0.01320	0.08581	0.10561	0.01980	0.53846	0	0
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
11284	1	0	0	0	0	0	0.00000	1	1	1.00000	...	0.00000	1.00000	0.00000	0.00000	0.00000	0.00000	0.00000	0.00000	0	0
44732	0	121	17	20	34	22	0.06335	807	24	4.50310	...	0.03346	0.01735	0.00000	0.01611	0.06568	0.08674	0.02726	0.52830	0	0
38158	0	6	1	1	1	2	-0.10606	89	5	4.02247	...	0.11236	0.04494	0.00000	0.01124	0.06742	0.13483	0.02247	0.28571	0	0
860	1	111	26	21	1	25	-0.03391	1092	48	3.83700	...	0.06868	0.04579	0.00183	0.02198	0.05586	0.14011	0.01282	0.28986	0	0
15795	1	36	10	13	4	3	0.03011	290	10	4.21724	...	0.03103	0.01034	0.00000	0.00690	0.10000	0.10345	0.01379	0.57143	0	0

	col	total	fake	true	prop_fake	total_normalized	prop_fake_normalized	importance_score
80	org_Reuters	21531	302	21229	0.014026	0.479554	-0.508959	0.494256
162	gpe_Washington	10768	1973	8795	0.183228	0.239833	-0.339757	0.289795
164	gpe_America	6650	5505	1145	0.827820	0.148114	0.304834	0.226474
3	person_Hillary	2943	2764	179	0.939178	0.065549	0.416192	0.240870
165	gpe_China	2611	492	2119	0.188434	0.058154	-0.334552	0.196353
...	...	...	...	...	...	...	...	...
154	org_Confederate	249	201	48	0.807229	0.005546	0.284243	0.144895
158	org_IRS	234	195	39	0.833333	0.005212	0.310348	0.157780
61	person_Reilly	217	206	11	0.949309	0.004833	0.426323	0.215578
50	person_Mugabe	192	4	188	0.020833	0.004276	-0.502152	0.253214
140	org_SPD	179	6	173	0.033520	0.003987	-0.489466	0.246726

	number	value	datetime_start	datetime_complete	duration	params_bagging_fraction	params_bagging_freq	params_feature_fraction	params_lambda_l1	params_lambda_l2	params_learning_rate	params_min_child_samples	params_num_leaves	state
1	1	0.999845	2024-08-13 18:51:21.834861	2024-08-13 18:51:25.769572	0 days 00:00:03.934711	0.972919	6	0.118526	4.329370e-07	4.473429e-07	0.133112	25	159	COMPLETE
2	2	0.999713	2024-08-13 18:51:25.772550	2024-08-13 18:51:29.750344	0 days 00:00:03.977794	0.362106	5	0.488751	4.258943e-06	1.982698e-05	0.037521	18	88	COMPLETE
0	0	0.999662	2024-08-13 18:51:19.022850	2024-08-13 18:51:21.831381	0 days 00:00:02.808531	0.638793	2	0.758795	3.332365e-08	6.245760e-01	0.711448	19	105	COMPLETE

	title	text	subject	date	fake	extracted_features	anonymized_text	num_entities	num_person_entities	num_org_entities	...	gpe_Georgia	gpe_Egypt	gpe_Benghazi	gpe_Italy	gpe_Sweden	gpe_Philippines	gpe_Oregon	gpe_Baghdad	gpe_Orlando	gpe_Illinois
15632	UPDATE: CATHOLIC UNIVERSITY THAT STATUE REMOVE...	This is a perfect example of the effort by lib...	politics	May 27, 2015	1	{'entities': [('American', 'NORP'), ('Indians'...	This is a perfect example of the effort by lib...	28	5	8	...	False	False	False	False	False	False	False	False	False	False
29490	Chelsea Manning criticizes Obama, draws Trump'...	WASHINGTON (Reuters) - Former U.S. military in...	politicsNews	January 26, 2017	0	{'entities': [('{RANDOM_ENTITY_GPE_0}', 'GPE')...	{RANDOM_ENTITY_GPE_0} ({RANDOM_ENTITY_ORG_1}) ...	51	23	6	...	False	False	False	False	False	False	False	False	False	False
22336	WAR ON WORDS: Facebook Censorship Widens, Webs...	21st Century Wire says The US media s neoMcCar...	US_News	December 3, 2016	1	{'entities': [('{RANDOM_ENTITY_GPE_0}', 'GPE')...	21st Century Wire says The {RANDOM_ENTITY_GPE_...	71	3	37	...	False	False	False	False	False	False	False	False	False	False
19487	DEMOCRATS AREN’T AFRAID Trump Will Be A Terrib...	The brilliant, outspoken and beautiful Tomi La...	left-news	Dec 7, 2016	1	{'entities': [('{RANDOM_ENTITY_PERSON_0}', 'PE...	The brilliant, outspoken and beautiful {RANDOM...	1	1	0	...	False	False	False	False	False	False	False	False	False	False
43007	Palestinian President Abbas says peace closer ...	NEW YORK (Reuters) - Palestinian President Mah...	worldnews	September 20, 2017	0	{'entities': [('{RANDOM_ENTITY_GPE_0}', 'GPE')...	{RANDOM_ENTITY_GPE_0} ({RANDOM_ENTITY_ORG_1}) ...	87	21	7	...	False	False	False	False	False	False	False	False	False	False
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
15714	HAPPY MOTHER’S DAY…Your First Grader Just Rate...	Rate your mom for Mother s Day sounds like the...	politics	May 11, 2015	1	{'entities': [('first', 'ORDINAL'), ('Ridgecre...	Rate your mom for {RANDOM_ENTITY_DATE_2} sound...	16	1	1	...	False	False	False	False	False	False	False	False	False	False
3926	Trump Goes COMPLETELY Off The Rails Over Hill...	The right wing is still harping on the FBI s d...	News	November 5, 2016	1	{'entities': [('{RANDOM_ENTITY_ORG_0}', 'ORG')...	The right wing is still harping on the {RANDOM...	23	13	4	...	False	False	False	False	False	False	False	False	False	False
35350	Chile election ends era of female presidents i...	SANTIAGO (Reuters) - When Chile s President Mi...	worldnews	December 19, 2017	0	{'entities': [('{RANDOM_ENTITY_ORG_0}', 'ORG')...	{RANDOM_ENTITY_ORG_0} ({RANDOM_ENTITY_ORG_1}) ...	101	29	11	...	False	False	False	False	False	False	False	False	False	False
3236	Charlie Sheen Sends SHOCKING Message About Tr...	This year was a rough one and many of us can t...	News	December 29, 2016	1	{'entities': [('{RANDOM_ENTITY_DATE_0}', 'DATE...	{RANDOM_ENTITY_DATE_0} was a rough one and man...	40	26	2	...	False	False	False	False	False	False	False	False	False	False
38093	Venezuela's ex-prosecutor wants Maduro tried a...	CARACAS (Reuters) - Venezuela s sacked former ...	worldnews	November 16, 2017	0	{'entities': [('{RANDOM_ENTITY_ORG_0}', 'ORG')...	CARACAS ({RANDOM_ENTITY_ORG_0}) - {RANDOM_ENTI...	37	14	5	...	False	False	False	False	False	False	False	False	False	False

	Column	Type
0	title	object
1	text	object
2	subject	object
3	date	object
4	fake	int64
5	extracted_features	object
6	anonymized_text	object
7	num_entities	int64
8	num_person_entities	int64
9	num_org_entities	int64
10	num_gpe_entities	int64
11	num_date_entities	int64
12	sentiment	float64
13	num_words	int64
14	num_sents	int64
15	avg_word_length	float64
16	lexical_diversity	float64
17	readability	float64
18	num_tokens	int64
19	char_count	int64
20	ner_density	float64
21	max_dep_depth	int64
22	avg_token_vector_norm	float64
23	avg_sent_length	float64
24	median_sent_length	float64
25	std_sent_length	float64
26	punctuation_density	float64
27	stop_words_ratio	float64
28	pos_ratio_PROPN	float64
29	pos_ratio_AUX	float64
30	pos_ratio_ADP	float64
31	pos_ratio_INTJ	float64
32	pos_ratio_PART	float64
33	pos_ratio_VERB	float64
34	pos_ratio_ADV	float64
35	pos_ratio_SYM	float64
36	pos_ratio_NOUN	float64
37	pos_ratio_ADJ	float64
38	pos_ratio_NUM	float64
39	pos_ratio_PRON	float64
40	pos_ratio_SPACE	float64
41	pos_ratio_X	float64
42	pos_ratio_SCONJ	float64
43	pos_ratio_DET	float64
44	pos_ratio_PUNCT	float64
45	pos_ratio_CCONJ	float64
46	verb_tense_ratio_past	float64
47	verb_tense_ratio_present	int64
48	verb_tense_ratio_future	int64
49	person_Trump	bool
50	person_Obama	bool
51	person_Clinton	bool
52	person_Hillary	bool
53	person_Sanders	bool
54	person_Cruz	bool
55	person_Putin	bool
56	person_Ryan	bool
57	person_Bush	bool
58	person_Flynn	bool
59	person_@realDonaldTrump	bool
60	person_McCain	bool
61	person_Obamacare	bool
62	person_Rubio	bool
63	person_Merkel	bool
64	person_Donald Trump s	bool
65	person_James Comey	bool
66	person_Tillerson	bool
67	person_Mike Pence	bool
68	person_Johnson	bool
69	person_McConnell	bool
70	person_Moore	bool
71	person_Kelly	bool
72	person_Assad	bool
73	person_Carson	bool
74	person_Hillary Clinton s	bool
75	person_Bannon	bool
76	person_Warren	bool
77	person_Donald	bool
78	person_Hariri	bool
79	person_Sean Spicer	bool
80	person_Mitt Romney	bool
81	person_Erdogan	bool
82	person_Jones	bool
83	person_Conway	bool
84	person_Biden	bool
85	person_Mattis	bool
86	person_Macron	bool
87	person_Graham	bool
88	person_Brown	bool
89	person_Xi	bool
90	person_Schumer	bool
91	person_Kushner	bool
92	person_Mnuchin	bool
93	person_Scalia	bool
94	person_Robert Mueller	bool
95	person_Soros	bool
96	person_Manafort	bool
97	person_Reagan	bool
98	person_Palin	bool
99	person_Mugabe	bool
100	person_Ivanka	bool
101	person_Kasich	bool
102	person_Gorsuch	bool
103	person_Don	bool
104	person_Jeff Sessions	bool
105	person_Kennedy	bool
106	person_Netanyahu	bool
107	person_Miller	bool
108	person_Abe	bool
109	person_Khan	bool
110	person_Reilly	bool
111	person_Stein	bool
112	person_Williams	bool
113	person_Reid	bool
114	person_Haley	bool
115	person_Smith	bool
116	person_Rice	bool
117	person_Nancy Pelosi	bool
118	person_Paul	bool
119	person_Ross	bool
120	person_Lee	bool
121	person_Weinstein	bool
122	person_Davis	bool
123	person_Bill	bool
124	person_Breitbart	bool
125	person_Carter	bool
126	person_King	bool
127	person_John Kerry	bool
128	person_Garland	bool
129	org_Reuters	bool
130	org_the White House	bool
131	org_Senate	bool
132	org_Congress	bool
133	org_FBI	bool
134	org_State	bool
135	org_EU	bool
136	org_CNN	bool
137	org_the State Department	bool
138	org_the Supreme Court	bool
139	org_GOP	bool
140	org_Fox News	bool
141	org_Facebook	bool
142	org_U.N.	bool
143	org_the New York Times	bool
144	org_Twitter	bool
145	org_CIA	bool
146	org_the Washington Post	bool
147	org_the United Nations	bool
148	org_ISIS	bool
149	org_the House of Representatives	bool
150	org_NATO	bool
151	org_the European Union	bool
152	org_the Republican Party	bool
153	org_the Democratic Party	bool
154	org_DNC	bool
155	org_Commission	bool
156	org_Comey	bool
157	org_NBC	bool
158	org_Pentagon	bool
159	org_MSNBC	bool
160	org_Army	bool
161	org_Fox	bool
162	org_Kremlin	bool
163	org_EPA	bool
164	org_Treasury	bool
165	org_the Department of Justice	bool
166	org_Breitbart News	bool
167	org_the U.N. Security Council	bool
168	org_Planned Parenthood	bool
169	org_the Clinton Foundation	bool
170	org_Medicaid	bool
171	org_Brexit	bool
172	org_al Qaeda	bool
173	org_Hezbollah	bool
174	org_ABC News	bool
175	org_Islam	bool
176	org_Politico	bool
177	org_Defense	bool
178	org_Capitol Hill	bool
179	org_Google	bool
180	org_Fed	bool
181	org_NFL	bool
182	org_Navy	bool
183	org_Group	bool
184	org_NSA	bool
185	org_Taliban	bool
186	org_ABC	bool
187	org_CBS	bool
188	org_WikiLeaks	bool
189	org_SPD	bool
190	org_the Democratic National Committee	bool
191	org_Mueller	bool
192	org_Justice Department	bool
193	org_the Wall Street Journal	bool
194	org_DOJ	bool
195	org_Daily Mail	bool
196	org_the Senate Intelligence Committee	bool
197	org_Getty Images	bool
198	org_NRA	bool
199	org_MOSCOW	bool
200	org_DHS	bool
201	org_the Department of Homeland Security	bool
202	org_LePage	bool
203	org_Confederate	bool
204	org_the Senate Judiciary Committee	bool
205	org_RNC	bool
206	org_Cabinet	bool
207	org_IRS	bool
208	org_Secret Service	bool
209	gpe_U.S.	bool
210	gpe_the United States	bool
211	gpe_Washington	bool
212	gpe_Russia	bool
213	gpe_America	bool
214	gpe_China	bool
215	gpe_Trump	bool
216	gpe_North Korea	bool
217	gpe_Iran	bool
218	gpe_Syria	bool
219	gpe_New York	bool
220	gpe_Iraq	bool
221	gpe_Mexico	bool
222	gpe_Britain	bool
223	gpe_Israel	bool
224	gpe_Texas	bool
225	gpe_Turkey	bool
226	gpe_Florida	bool
227	gpe_Germany	bool
228	gpe_Moscow	bool
229	gpe_Myanmar	bool
230	gpe_California	bool
231	gpe_Saudi Arabia	bool
232	gpe_London	bool
233	gpe_Beijing	bool
234	gpe_North Carolina	bool
235	gpe_Japan	bool
236	gpe_France	bool
237	gpe_Afghanistan	bool
238	gpe_Paris	bool
239	gpe_Virginia	bool
240	gpe_India	bool
241	gpe_Jerusalem	bool
242	gpe_Ohio	bool
243	gpe_Cuba	bool
244	gpe_Ukraine	bool
245	gpe_Yemen	bool
246	gpe_Libya	bool
247	gpe_Chicago	bool
248	gpe_Brussels	bool
249	gpe_Pyongyang	bool
250	gpe_Canada	bool
251	gpe_Australia	bool
252	gpe_Michigan	bool
253	gpe_Pakistan	bool
254	gpe_Puerto Rico	bool
255	gpe_Iowa	bool
256	gpe_Lebanon	bool
257	gpe_Bangladesh	bool
258	gpe_Arizona	bool
259	gpe_Taiwan	bool
260	gpe_Hollywood	bool
261	gpe_Spain	bool
262	gpe_UK	bool
263	gpe_Alabama	bool
264	gpe_Tehran	bool
265	gpe_Berlin	bool
266	gpe_New Jersey	bool
267	gpe_Wisconsin	bool
268	gpe_New York City	bool
269	gpe_D.C.	bool
270	gpe_Venezuela	bool
271	gpe_Kansas	bool
272	gpe_Pennsylvania	bool
273	gpe_Vietnam	bool
274	gpe_Poland	bool
275	gpe_Flint	bool
276	gpe_New Hampshire	bool
277	gpe_Catalonia	bool
278	gpe_Ireland	bool
279	gpe_Georgia	bool
280	gpe_Egypt	bool
281	gpe_Benghazi	bool
282	gpe_Italy	bool
283	gpe_Sweden	bool
284	gpe_Philippines	bool
285	gpe_Oregon	bool
286	gpe_Baghdad	bool
287	gpe_Orlando	bool
288	gpe_Illinois	bool

	Unnamed: 0	title	text	subject	date	fake	extracted_features	anonymized_text	num_entities	num_person_entities	...	gpe_Pyongyang	gpe_Canada	gpe_Australia	gpe_Michigan	gpe_Pakistan	gpe_Puerto Rico	gpe_Iowa	gpe_Lebanon	gpe_Bangladesh	gpe_Arizona
11	12038	WHERE’S HILLARY? CLINTON SPOTTED Dining Alone		politics	Dec 30, 2016	1	{'entities': [], 'entity_counts': {}, 'person_...		0	0	...	False	False	False	False	False	False	False	False	False	False
41	12750	WHOA! RUSH LIMBAUGH RIPS Into Republicans Who ...		politics	Oct 12, 2016	1	{'entities': [], 'entity_counts': {}, 'person_...		0	0	...	False	False	False	False	False	False	False	False	False	False
177	13008	HILLARY TAKES CREDIT For The Arab Spring Disas...		politics	Sep 15, 2016	1	{'entities': [], 'entity_counts': {}, 'person_...		0	0	...	False	False	False	False	False	False	False	False	False	False
324	11478	CHILLING! FOX REPORTER JAMES ROSEN Recounts Be...		politics	Mar 6, 2017	1	{'entities': [], 'entity_counts': {}, 'person_...		0	0	...	False	False	False	False	False	False	False	False	False	False
400	11854	LIVE FEED: INAUGURATION 2017!		politics	Jan 20, 2017	1	{'entities': [], 'entity_counts': {}, 'person_...		0	0	...	False	False	False	False	False	False	False	False	False	False
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
44632	12183	TUCKER CARLSON DESTROYS Smug Elector Who Refus...	https://youtu.be/7oOhwHG2Gb4	politics	Dec 9, 2016	1	{'entities': [], 'entity_counts': {}, 'person_...	https://youtu.be/7oOhwHG2Gb4	0	0	...	False	False	False	False	False	False	False	False	False	False
44782	19115	TUCKER CARLSON Defends Trump On Sweden Comment...		left-news	Feb 20, 2017	1	{'entities': [], 'entity_counts': {}, 'person_...		0	0	...	False	False	False	False	False	False	False	False	False	False
44868	19118	AWESOME! HISPANIC TRUMP SUPPORTER Rips Into Pr...		left-news	Feb 19, 2017	1	{'entities': [], 'entity_counts': {}, 'person_...		0	0	...	False	False	False	False	False	False	False	False	False	False
44878	19769	DEMOCRAT OPERATIVES Caught Planning To Bully W...		left-news	Oct 20, 2016	1	{'entities': [], 'entity_counts': {}, 'person_...		0	0	...	False	False	False	False	False	False	False	False	False	False
44893	11284	UNREAL! CBS’S TED KOPPEL Tells Sean Hannity He...		politics	Mar 27, 2017	1	{'entities': [], 'entity_counts': {}, 'person_...		0	0	...	False	False	False	False	False	False	False	False	False	False

	Unnamed: 0	title	text	subject	date	fake	extracted_features	anonymized_text	num_entities	num_person_entities	...	gpe_Pyongyang	gpe_Canada	gpe_Australia	gpe_Michigan	gpe_Pakistan	gpe_Puerto Rico	gpe_Iowa	gpe_Lebanon	gpe_Bangladesh	gpe_Arizona
0	22216	Ben Stein Calls Out 9th Circuit Court: Committ...	21st Century Wire says Ben Stein, reputable pr...	US_News	February 13, 2017	1	{'entities': [('{RANDOM_ENTITY_PERSON_0}', 'PE...	21st Century Wire says {RANDOM_ENTITY_PERSON_0...	13	6	...	False	False	False	False	False	False	False	False	False	False
1	27917	Trump drops Steve Bannon from National Securit...	WASHINGTON (Reuters) - U.S. President Donald T...	politicsNews	April 5, 2017	0	{'entities': [('{RANDOM_ENTITY_GPE_0}', 'GPE')...	{RANDOM_ENTITY_GPE_0} ({RANDOM_ENTITY_ORG_1}) ...	114	49	...	False	False	False	False	False	False	False	False	False	False
2	25007	Puerto Rico expects U.S. to lift Jones Act shi...	(Reuters) - Puerto Rico Governor Ricardo Rosse...	politicsNews	September 27, 2017	0	{'entities': [('{RANDOM_ENTITY_ORG_0}', 'ORG')...	({RANDOM_ENTITY_ORG_0}) - {RANDOM_ENTITY_GPE_1...	26	5	...	False	False	False	False	False	True	False	False	False	False
3	1377	OOPS: Trump Just Accidentally Confirmed He Le...	On Monday, Donald Trump once again embarrassed...	News	May 22, 2017	1	{'entities': [('{RANDOM_ENTITY_DATE_0}', 'DATE...	On {RANDOM_ENTITY_DATE_0}, {RANDOM_ENTITY_PERS...	24	8	...	False	False	False	False	False	False	False	False	False	False
4	32476	Donald Trump heads for Scotland to reopen a go...	GLASGOW, Scotland (Reuters) - Most U.S. presid...	politicsNews	June 24, 2016	0	{'entities': [('{RANDOM_ENTITY_GPE_0}', 'GPE')...	GLASGOW, {RANDOM_ENTITY_GPE_0} ({RANDOM_ENTITY...	75	10	...	False	False	False	False	False	False	False	False	False	False
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
44893	11284	UNREAL! CBS’S TED KOPPEL Tells Sean Hannity He...		politics	Mar 27, 2017	1	{'entities': [], 'entity_counts': {}, 'person_...		0	0	...	False	False	False	False	False	False	False	False	False	False
44894	44732	PM May seeks to ease Japan's Brexit fears duri...	LONDON/TOKYO (Reuters) - British Prime Ministe...	worldnews	August 29, 2017	0	{'entities': [('{RANDOM_ENTITY_GPE_0}', 'GPE')...	{RANDOM_ENTITY_GPE_0}/TOKYO ({RANDOM_ENTITY_OR...	121	17	...	False	False	False	False	False	False	False	False	False	False
44895	38158	Merkel: Difficult German coalition talks can r...	BERLIN (Reuters) - Chancellor Angela Merkel sa...	worldnews	November 16, 2017	0	{'entities': [('{RANDOM_ENTITY_GPE_0}', 'GPE')...	{RANDOM_ENTITY_GPE_0} ({RANDOM_ENTITY_ORG_1}) ...	6	1	...	False	False	False	False	False	False	False	False	False	False
44896	860	Trump Stole An Idea From North Korean Propaga...	Jesus fcking Christ our President is a moron...	News	July 14, 2017	1	{'entities': [('{RANDOM_ENTITY_PERSON_0}', 'PE...	{RANDOM_ENTITY_PERSON_0} f*cking Christ our Pr...	111	26	...	False	False	False	False	False	False	False	False	False	False
44897	15795	BREAKING: HILLARY CLINTON’S STATE DEPARTMENT G...	IF SHE S NOT TOAST NOW THEN WE RE IN BIGGER TR...	politics	Apr 23, 2015	1	{'entities': [('{RANDOM_ENTITY_PERSON_0}', 'PE...	IF SHE S NOT TOAST NOW THEN WE RE IN BIGGER TR...	36	10	...	False	False	False	False	False	False	False	False	False	False

	feature	importance	importance_percentage	mean_abs_shap	shap_importance_percentage
5	pos_ratio_PUNCT	1259	3.035344	4.402535	10.296663
9	ner_density	1204	2.902744	2.211655	5.172626
2	verb_tense_ratio_past	1383	3.334298	2.019577	4.723392
3	avg_token_vector_norm	1364	3.288490	1.865964	4.364123
16	punctuation_density	1123	2.707459	1.830244	4.280579
1	std_sent_length	1397	3.368051	1.689886	3.952311
8	pos_ratio_ADV	1246	3.004002	1.572365	3.677453
30	num_date_entities	746	1.798544	1.551266	3.628105
25	num_gpe_entities	884	2.131250	1.450836	3.393219
13	avg_word_length	1150	2.772554	1.380218	3.228058
23	avg_sent_length	983	2.369931	1.142517	2.672123
15	pos_ratio_PRON	1130	2.724336	1.127723	2.637522
4	pos_ratio_SPACE	1266	3.052220	1.093518	2.557523
26	median_sent_length	881	2.124018	1.081556	2.529546
22	pos_ratio_PROPN	1008	2.430204	1.071308	2.505578
41	gpe_U.S.	73	0.175997	1.050933	2.457926
28	num_person_entities	835	2.013115	0.928912	2.172543
6	pos_ratio_PART	1252	3.018468	0.849034	1.985723
40	person_Obama	86	0.207339	0.798815	1.868271
43	person_Trump	54	0.130189	0.781697	1.828234
29	num_org_entities	802	1.933555	0.725261	1.696243
32	num_words	585	1.410386	0.652742	1.526636
24	pos_ratio_NUM	896	2.160181	0.648339	1.516337
56	person_@realDonaldTrump	41	0.098848	0.598985	1.400909
12	pos_ratio_SCONJ	1173	2.828005	0.575131	1.345118

	Fake_Ratio	Total_Count
subject
Government News	1.0	1205
Middle-east	1.0	616
News	1.0	7343
US_News	1.0	640
left-news	1.0	3478
politics	1.0	5225
politicsNews	0.0	9080
worldnews	0.0	8251

	raw_correlation	abs_correlation
pos_ratio_PRON	0.45	0.45
pos_ratio_ADV	0.45	0.45
verb_tense_ratio_past	-0.45	0.45
std_sent_length	0.41	0.41
median_sent_length	-0.39	0.39
avg_sent_length	-0.32	0.32
readability	0.31	0.31
ner_density	-0.31	0.31
pos_ratio_X	0.29	0.29
avg_word_length	-0.28	0.28
stop_words_ratio	0.25	0.25
lexical_diversity	-0.20	0.20
punctuation_density	0.19	0.19
avg_token_vector_norm	-0.14	0.14
pos_ratio_PART	-0.13	0.13

	raw_correlation	abs_correlation
pos_ratio_INTJ	0.36	0.36
pos_ratio_NOUN	-0.32	0.32
pos_ratio_SCONJ	0.30	0.30
pos_ratio_ADP	-0.26	0.26
pos_ratio_SPACE	0.20	0.20
pos_ratio_SYM	0.18	0.18
pos_ratio_DET	-0.18	0.18
pos_ratio_PUNCT	-0.12	0.12
pos_ratio_ADJ	-0.12	0.12
pos_ratio_AUX	0.10	0.10

	0	1	accuracy	macro avg	weighted avg

precision	0.972950	0.976971	0.975021	0.974960	0.975026
recall	0.975476	0.974595	0.975021	0.975035	0.975021
f1-score	0.974211	0.975781	0.975021	0.974996	0.975022
support	3466.000000	3700.000000	0.975021	7166.000000	7166.000000

entity	@realDonaldTrump	Assad	Bannon	Biden	Brown	Bush	Carson	Clinton	Comey	Conway	...	Texas	Trump	Turkey	U.S.	UK	Ukraine	Virginia	Washington	Yemen	the United States
entity_type	person	person	person	person	person	person	person	person	person	person	...	gpe	gpe	gpe	gpe	gpe	gpe	gpe	gpe	gpe	gpe
date
2015-03	0.000000	0.000000	0.000000	0.055556	0.055556	0.000000	0.000000	0.166667	0.000000	0.000000	...	0.000000	0.000000	0.000000	0.200000	0.000000	0.000000	0.000000	0.600000	0.000000	0.000000
2015-04	0.000000	0.000000	0.000000	0.008333	0.054167	0.016667	0.012500	0.145833	0.000000	0.004167	...	0.030879	0.000000	0.004751	0.161520	0.009501	0.009501	0.009501	0.054632	0.007126	0.149644
2015-05	0.000000	0.000000	0.000000	0.020325	0.024390	0.028455	0.012195	0.146341	0.000000	0.000000	...	0.060526	0.000000	0.007895	0.197368	0.005263	0.002632	0.002632	0.060526	0.005263	0.089474
2015-06	0.014440	0.000000	0.000000	0.003610	0.032491	0.014440	0.003610	0.111913	0.000000	0.000000	...	0.062937	0.006993	0.003497	0.132867	0.010490	0.000000	0.010490	0.066434	0.000000	0.104895
2015-07	0.086093	0.006623	0.000000	0.000000	0.023179	0.039735	0.013245	0.052980	0.000000	0.000000	...	0.021429	0.026190	0.002381	0.166667	0.004762	0.000000	0.014286	0.050000	0.004762	0.095238
2015-08	0.082317	0.003049	0.006098	0.024390	0.033537	0.027439	0.006098	0.112805	0.000000	0.000000	...	0.054167	0.025000	0.000000	0.133333	0.008333	0.000000	0.016667	0.041667	0.000000	0.087500
2015-09	0.062500	0.031250	0.000000	0.026786	0.008929	0.013393	0.026786	0.071429	0.000000	0.004464	...	0.057143	0.014286	0.007143	0.125000	0.000000	0.000000	0.007143	0.060714	0.000000	0.089286
2015-10	0.046154	0.007692	0.000000	0.035897	0.010256	0.038462	0.023077	0.125641	0.007692	0.000000	...	0.024793	0.000000	0.002755	0.129477	0.008264	0.008264	0.016529	0.044077	0.005510	0.079890
2015-11	0.048701	0.016234	0.000000	0.003247	0.006494	0.025974	0.019481	0.084416	0.003247	0.009740	...	0.039823	0.017699	0.026549	0.130531	0.006637	0.002212	0.002212	0.053097	0.002212	0.084071
2015-12	0.062323	0.008499	0.000000	0.011331	0.002833	0.016997	0.011331	0.048159	0.011331	0.002833	...	0.008032	0.036145	0.008032	0.150602	0.006024	0.002008	0.008032	0.030120	0.008032	0.090361
2016-01	0.100234	0.000937	0.001405	0.011241	0.006557	0.043560	0.010304	0.067447	0.000000	0.000000	...	0.043162	0.040562	0.004160	0.136245	0.007800	0.000000	0.010920	0.051482	0.002080	0.113885
2016-02	0.091929	0.000362	0.000000	0.008686	0.006515	0.042345	0.022801	0.064423	0.000724	0.000362	...	0.061252	0.045372	0.002722	0.129310	0.002269	0.001815	0.015880	0.050363	0.001361	0.110254
2016-03	0.123330	0.002784	0.000000	0.003898	0.008352	0.021158	0.007795	0.065980	0.001949	0.001114	...	0.046479	0.080819	0.001734	0.136663	0.004509	0.003469	0.010753	0.054804	0.002081	0.096774
2016-04	0.112870	0.004060	0.000812	0.003654	0.006902	0.018676	0.006496	0.092570	0.002030	0.000406	...	0.050219	0.058020	0.003901	0.146758	0.000975	0.005851	0.016090	0.054120	0.002925	0.105802
2016-05	0.122956	0.000000	0.000654	0.002943	0.007521	0.016351	0.005559	0.095814	0.000981	0.000327	...	0.027764	0.087189	0.001948	0.146128	0.009255	0.001461	0.021919	0.051632	0.001461	0.116902
2016-06	0.123615	0.002424	0.000693	0.005194	0.007964	0.013504	0.001039	0.114266	0.003463	0.000000	...	0.024861	0.070724	0.003429	0.146164	0.012002	0.000857	0.011144	0.045435	0.003429	0.117874
2016-07	0.107921	0.001038	0.001038	0.005189	0.010031	0.015220	0.004151	0.124524	0.010723	0.000346	...	0.038732	0.067488	0.004108	0.142606	0.004695	0.006455	0.022887	0.047535	0.000587	0.110915
2016-08	0.116339	0.000645	0.006123	0.004834	0.004834	0.013858	0.003223	0.139542	0.003867	0.005801	...	0.014475	0.103136	0.002413	0.139928	0.003619	0.007238	0.025332	0.043426	0.001206	0.092280
2016-09	0.111041	0.003490	0.000635	0.003173	0.003807	0.014277	0.002538	0.138642	0.003173	0.006980	...	0.015648	0.085086	0.005379	0.143765	0.009291	0.006357	0.013692	0.053301	0.002934	0.105623
2016-10	0.114791	0.005976	0.000747	0.006225	0.000996	0.016434	0.001245	0.154382	0.012201	0.003735	...	0.016835	0.118807	0.005772	0.142857	0.001924	0.008658	0.019240	0.054834	0.003367	0.089466
2016-11	0.112899	0.004734	0.010651	0.001657	0.002130	0.020118	0.005207	0.104615	0.008994	0.009467	...	0.016196	0.088560	0.004135	0.141626	0.008959	0.013784	0.012750	0.058236	0.001723	0.095107
2016-12	0.123891	0.004006	0.009442	0.002289	0.008298	0.023176	0.008870	0.064664	0.004578	0.009442	...	0.017268	0.097724	0.005102	0.153846	0.004710	0.009027	0.010204	0.063579	0.003925	0.100078
2017-01	0.123987	0.003577	0.007630	0.004053	0.004530	0.017883	0.004530	0.043157	0.004769	0.008345	...	0.011667	0.093056	0.002500	0.160000	0.006389	0.007222	0.007222	0.077778	0.011944	0.105833
2017-02	0.123184	0.003196	0.021209	0.000581	0.004067	0.019175	0.003486	0.030215	0.001162	0.015398	...	0.012123	0.082785	0.004503	0.162106	0.005542	0.012470	0.011431	0.066851	0.022515	0.115691
2017-03	0.115934	0.002157	0.008088	0.001348	0.004044	0.015638	0.004583	0.028310	0.015368	0.007819	...	0.021466	0.108808	0.006662	0.165433	0.005922	0.008512	0.014064	0.061806	0.010363	0.101406
2017-04	0.120399	0.028896	0.009976	0.000688	0.000688	0.012728	0.007568	0.033712	0.005504	0.003440	...	0.013785	0.081595	0.005589	0.157973	0.002981	0.003726	0.007452	0.069300	0.004471	0.100224
2017-05	0.106183	0.002590	0.004856	0.000647	0.002590	0.012302	0.005827	0.048559	0.071544	0.008093	...	0.016997	0.098206	0.005666	0.154863	0.001889	0.004249	0.012748	0.069877	0.006610	0.091124
2017-06	0.122746	0.005694	0.003164	0.003796	0.002214	0.009807	0.004113	0.037963	0.045555	0.008225	...	0.013211	0.094476	0.002002	0.157326	0.013611	0.004003	0.022818	0.070456	0.008407	0.091673
2017-07	0.118481	0.004522	0.005125	0.001206	0.002713	0.010853	0.007235	0.056376	0.010552	0.005427	...	0.011471	0.081114	0.003277	0.157722	0.006555	0.014338	0.015158	0.058583	0.004916	0.089717
2017-08	0.138767	0.001836	0.017254	0.001836	0.003671	0.012482	0.005140	0.040749	0.005874	0.002937	...	0.023238	0.072339	0.004123	0.143178	0.004873	0.008996	0.042354	0.059220	0.004123	0.095952
2017-09	0.124822	0.017099	0.007125	0.001425	0.005985	0.010544	0.003420	0.025363	0.003990	0.000570	...	0.013745	0.024247	0.029189	0.118919	0.008340	0.009730	0.007876	0.043707	0.007104	0.094054
2017-10	0.115757	0.014124	0.003877	0.001662	0.004985	0.009970	0.002215	0.042371	0.008031	0.001385	...	0.008389	0.026907	0.026749	0.132953	0.012662	0.006173	0.007281	0.056347	0.006331	0.096233
2017-11	0.114820	0.017623	0.004539	0.005073	0.003204	0.008278	0.002403	0.042190	0.006943	0.001335	...	0.010010	0.029545	0.021311	0.128834	0.010010	0.011947	0.008072	0.043429	0.017275	0.092347
2017-12	0.129933	0.013965	0.003339	0.000607	0.001214	0.010018	0.003339	0.034305	0.010929	0.001214	...	0.005868	0.028975	0.022006	0.135155	0.012654	0.013204	0.005868	0.050614	0.012654	0.094260
2018-02	0.137931	0.000000	0.000000	0.000000	0.000000	0.000000	0.017241	0.051724	0.000000	0.000000	...	0.019231	0.076923	0.000000	0.192308	0.000000	0.000000	0.000000	0.019231	0.000000	0.096154

	Reuters %	N Reuters articles	N non-Reuters articles
fake
0	0.997980	17296	35
1	0.014434	267	18231

	fake	DROPPED_Reuters	text	anonymized_text	anonymized_text_bert-base-uncased_used_text
0	1	False	Meryl Streep and Viola Davis weren t the only ...	[PERSON0] and [PERSON1] weren t the only ones ...	[PERSON0] and [PERSON1] weren t the only ones ...
1	1	False	Never has a movie role so perfectly described ...	Never has a movie role so perfectly described ...	never has a movie role so perfectly described ...
2	0	True	A leader of the smaller Free Democrats (FDP) a...	A leader of the smaller Free Democrats ([ORG29...	a leader of the smaller free democrats ( [ org...
3	1	False	Does this president know American history or d...	Does this president know American history or d...	Does this president know American history or d...
4	0	True	Gholamhossein Mohseni Ejei named the convicted...	[PERSON0] named the convicted man as [PERSON1]...	[PERSON0] named the convicted man as [PERSON1]...
...	...	...	...	...	...
35833	1	False	Nobody can accuse the GOP of having it all tog...	Nobody can accuse the [ORG0] of having it all ...	Nobody can accuse the [ORG0] of having it all ...
35834	0	True	Tunisia late on Sunday suspended flights from ...	[GPE0] late on [DATE1] suspended flights from ...	[GPE0] late on [DATE1] suspended flights from ...
35835	0	True	Lavrov said he had last had a telephone conver...	[ORG0] said he had last had a telephone conver...	[ORG0] said he had last had a telephone conver...
35836	0	True	Sworn in days after the overthrow of Robert Mu...	Sworn in [DATE0] after the overthrow of [PERSO...	sworn in [ date0 ] after the overthrow of [ pe...
35837	1	False	This picture is representative of the violence...	This picture is representative of the violence...	This picture is representative of the violence...

	min_samples	alpha	silhouette_score	n_clusters	noise_proportion
149	95	1.814286	0.314928	2	0.094625
151	95	2.500000	0.314928	2	0.094625
144	95	0.100000	0.314928	2	0.094625
145	95	0.442857	0.314928	2	0.094625
146	95	0.785714	0.314928	2	0.094625
...	...	...	...	...	...
28	20	1.471429	-0.047804	3	0.547875
27	20	1.128571	-0.047804	3	0.547875
26	20	0.785714	-0.047804	3	0.547875
25	20	0.442857	-0.047804	3	0.547875
24	20	0.100000	-0.047804	3	0.547875

1.4 Entity Analysis¶

1.4.1 PCA¶

1.5 Temporal Analysis¶

3.0 Heuristics Based Model¶

3.1 Feature Selection¶

3.3 Feature Analysis¶

3.3.1 pos_ratio_PUNCT¶

3.3.2 ner_DENSITY¶

3.3.3 verb_tense_ratio_past¶

3.3.4 avg_token_vector_norm¶

3.3.5 punctuation_density¶

3.3.6 std_sent_length¶

3.3.7 pos_ratio_ADV¶

{INTRO AND DESCRIPTION}¶

Dataset Summary¶

Semantic Analysis etc.¶

Lexical Features¶

Syntactic Features¶

Semantic Features¶

Structural Features¶

Temporal Features¶

Entity Features¶

Building a simple model based on semantic features¶

Entity Analysis¶

Most Popular Entities¶

Only Entities in "Fake News" Articles¶

Summary Of Entities by Target Variable¶

Sentiment:¶

Embedings and Building a DNN Model¶

Non-representivie dataset¶

Temporal Biases¶

Annomyizing text samples¶

Example of preprocessing pipeline:¶

Generating and Analyzing Embeddings¶

Clustering¶

BERT embeddings reduced with t-SNE based on target variable ("fake"/non-Reuters):¶

Clustering, "best" configuration with 2 final clusters:¶

Raw Text Clusters¶

Processed Text Clusters¶

Anonymized Text Clusters¶

Topic Analysis¶

Cluster Centroid Summarization¶